diff --git a/CHANGES-3.3 b/CHANGES-3.3 index 3a1c8fe293fef02784614b75a1beb59b7ef5db93..6378c7be678827dad61218dafadc6ac92903774e 100644 --- a/CHANGES-3.3 +++ b/CHANGES-3.3 @@ -10,13 +10,22 @@ the history of the 3.2 series (2018-05 - 2022-08). # 3.3.14 (unreleased) This release contains contributions from (alphabetically by given name): - - Nobody yet! + - Adriaan de Groot + - TNE + - vincent PENVERN ## Core ## - - Nothing yet! + - The Python bindings have been re-organized (source) and made more + consistent. At least one valid Python program would work with + the Boost::Python bindings, but not the pybind11 bindings. A + memory-corruption problem in the Boost::Python bindings was resolved. ## Modules ## - - Nothing yet! + - *partition* module stores a global storage value in luksPassphrase, + for later modules that need to manipulate the encrypted partition. + (thanks vincent, #2424) + - *partition* module no longer clear (unmounts) a Ventoy device. + (thanks TNE, #2427) # 3.3.13 (2024-12-31) @@ -35,7 +44,7 @@ This release contains contributions from (alphabetically by given name): - Fewer compile warnings with most-recent Qt versions. - Support systemd and consolekit block-suspend, not just KDE Plasma block-suspend, during installation. (thanks Jakob, #2404) - + ## Modules ## - *dracut* module has more freedom to specify program options. (thanks Simon, #2401) - *partition* module improved some user-visible messages. (thanks Masato, #2412) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0aa69f1d03a31abaf1ce1dc788875292cb0cdddf..4c534af0409847bbb366cfc62643b8810dd43911 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -93,6 +93,8 @@ option(BUILD_SCHEMA_TESTING "Enable schema-validation-tests" ON) # Options for the calamares executable option(BUILD_CRASH_REPORTING "Enable crash reporting with KCrash." ON) +option(DEBUG_SANITIZERS "Enable sanitizers and Debug build type" OFF) + # Possible debugging flags are: # - DEBUG_TIMEZONES draws latitude and longitude lines on the timezone # widget and enables chatty debug logging, for dealing with the timezone @@ -272,6 +274,9 @@ set(CMAKE_C_FLAGS_RELWITHDEBINFO "-O2 -g") set(CMAKE_SHARED_LINKER_FLAGS "-Wl,--no-undefined -Wl,--fatal-warnings ${CMAKE_SHARED_LINKER_FLAGS}") # If no build type is set, pick a reasonable one +if(DEBUG_SANITIZERS) + set(CMAKE_BUILD_TYPE DEBUG) +endif() if(NOT CMAKE_BUILD_TYPE) if(CALAMARES_RELEASE_MODE) set(CMAKE_BUILD_TYPE "RelWithDebInfo") @@ -338,6 +343,10 @@ if(CMAKE_COMPILER_IS_GNUCXX) message(STATUS "Found GNU g++ ${CMAKE_CXX_COMPILER_VERSION}, enabling colorized error messages.") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdiagnostics-color=auto") endif() + if(DEBUG_SANITIZERS) + message(STATUS "Setting up sanitizers") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined -fsanitize=address -g -O1") + endif() endif() ### DEPENDENCIES diff --git a/src/calamares/testmain.cpp b/src/calamares/testmain.cpp index f40cf3629e4fcca23c187f78d021a80362eae936..c9d89c0c2b77dffe198d008eabc2a9c071d167a6 100644 --- a/src/calamares/testmain.cpp +++ b/src/calamares/testmain.cpp @@ -35,9 +35,9 @@ // - QML support #ifdef WITH_PYTHON #ifdef WITH_PYBIND11 -#include "python/PythonJob.h" +#include "pybind11/PythonJob.h" #else -#include "PythonJob.h" +#include "pyboost/PythonJob.h" #endif #endif #ifdef WITH_QML @@ -500,7 +500,8 @@ main( int argc, char* argv[] ) #endif cDebug() << "Calamares module-loader testing" << module.moduleName(); - Calamares::Module* m = load_module( module ); + std::unique_ptr<Calamares::Module> m( load_module( module ) ); + std::unique_ptr<Calamares::ModuleManager> modulemanager; if ( !m ) { cError() << "Could not load module" << module.moduleName(); @@ -527,9 +528,9 @@ main( int argc, char* argv[] ) } (void)new Calamares::Branding( module.m_branding ); - auto* modulemanager = new Calamares::ModuleManager( QStringList(), nullptr ); + modulemanager = std::make_unique<Calamares::ModuleManager>( QStringList(), nullptr ); (void)Calamares::ViewManager::instance( mainWindow ); - modulemanager->addModule( m ); + modulemanager->addModule( m.release() ); // Transfers ownership } if ( !m->isLoaded() ) diff --git a/src/libcalamares/CMakeLists.txt b/src/libcalamares/CMakeLists.txt index 1467e85faaad7537d8a864ebdd259fe856bd9e92..57a0452494f36b54f5b4c182a0f0ef47e6f6dd01 100644 --- a/src/libcalamares/CMakeLists.txt +++ b/src/libcalamares/CMakeLists.txt @@ -128,12 +128,15 @@ endif() # if(WITH_PYTHON) if(WITH_PYBIND11) - target_sources(calamares PRIVATE python/Api.cpp python/PythonJob.cpp) + target_include_directories(calamares PRIVATE pybind11) + target_sources(calamares PRIVATE pybind11/Api.cpp pybind11/PythonJob.cpp) target_link_libraries(calamares PRIVATE Python::Python pybind11::headers) else() - target_sources(calamares PRIVATE PythonHelper.cpp PythonJob.cpp PythonJobApi.cpp) + target_include_directories(calamares PRIVATE pyboost) + target_sources(calamares PRIVATE pyboost/PythonHelper.cpp pyboost/PythonJob.cpp pyboost/PythonJobApi.cpp) target_link_libraries(calamares PRIVATE Python::Python Boost::python) endif() + target_sources(calamares PRIVATE python/Api.cpp python/Variant.cpp) endif() ### OPTIONAL GeoIP XML support diff --git a/src/libcalamares/ProcessJob.cpp b/src/libcalamares/ProcessJob.cpp index d3d76afb8763e64a11ef7c57989148f28753101f..3d938d5b9b025e7214d463b1103495f0af866147 100644 --- a/src/libcalamares/ProcessJob.cpp +++ b/src/libcalamares/ProcessJob.cpp @@ -31,7 +31,7 @@ ProcessJob::ProcessJob( const QString& command, { } -ProcessJob::~ProcessJob() {} +ProcessJob::~ProcessJob() = default; QString ProcessJob::prettyName() const diff --git a/src/libcalamares/ProcessJob.h b/src/libcalamares/ProcessJob.h index 19d886bb5bb8abba1b116f655c21789d3fc2d9ad..2e44ba38ad52a6e7b3d79dd8077e4370a8f8cb71 100644 --- a/src/libcalamares/ProcessJob.h +++ b/src/libcalamares/ProcessJob.h @@ -19,11 +19,11 @@ namespace Calamares { -class ProcessJob : public Job +class DLLEXPORT ProcessJob : public Job { Q_OBJECT public: - explicit DLLEXPORT ProcessJob( const QString& command, + explicit ProcessJob( const QString& command, const QString& workingPath, bool runInChroot = false, std::chrono::seconds secondsTimeout = std::chrono::seconds( 30 ), diff --git a/src/libcalamares/pybind11/Api.cpp b/src/libcalamares/pybind11/Api.cpp new file mode 100644 index 0000000000000000000000000000000000000000..61829e332bbd84422cdec12949975937dbc5006b --- /dev/null +++ b/src/libcalamares/pybind11/Api.cpp @@ -0,0 +1,322 @@ +/* === This file is part of Calamares - <https://calamares.io> === + * + * SPDX-FileCopyrightText: 2014 Teo Mrnjavac <teo@kde.org> + * SPDX-FileCopyrightText: 2017-2020, 2023 Adriaan de Groot <groot@kde.org> + * SPDX-License-Identifier: GPL-3.0-or-later + * + * Calamares is Free Software: see the License-Identifier above. + * + */ +#include "Api.h" + +#include "Pybind11Helpers.h" +#include "PythonJob.h" + +#include "GlobalStorage.h" +#include "JobQueue.h" +#include "compat/Variant.h" +#include "locale/Global.h" +#include "python/Variant.h" +#include "utils/Logger.h" +#include "utils/RAII.h" +#include "utils/Runner.h" +#include "utils/String.h" +#include "utils/System.h" +#include "utils/Yaml.h" + +#include <QCoreApplication> +#include <QDir> +#include <QStandardPaths> + +namespace py = pybind11; + +/** @namespace + * + * Helper functions for converting Python (pybind11) types to Qt types. + */ +namespace +{ + +QVariantList variantListFromPyList( const Calamares::Python::List& list ); +QVariantMap variantMapFromPyDict( const Calamares::Python::Dictionary& dict ); + +QVariant +variantFromPyObject( const py::handle& o ) +{ + if ( py::isinstance< Calamares::Python::Dictionary >( o ) ) + { + return variantMapFromPyDict( py::cast< Calamares::Python::Dictionary >( o ) ); + } + else if ( py::isinstance< Calamares::Python::List >( o ) ) + { + return variantListFromPyList( py::cast< Calamares::Python::List >( o ) ); + } + else if ( py::isinstance< py::int_ >( o ) ) + { + return QVariant( qlonglong( py::cast< py::int_ >( o ) ) ); + } + else if ( py::isinstance< py::float_ >( o ) ) + { + return QVariant( double( py::cast< py::float_ >( o ) ) ); + } + else if ( py::isinstance< py::str >( o ) ) + { + return QVariant( QString::fromStdString( std::string( py::str( o ) ) ) ); + } + else if ( py::isinstance< py::bool_ >( o ) ) + { + return QVariant( bool( py::cast< py::bool_ >( o ) ) ); + } + + return QVariant(); +} + +QVariantList +variantListFromPyList( const Calamares::Python::List& list ) +{ + QVariantList l; + for ( const auto item : list ) + { + l.append( variantFromPyObject( item ) ); + } + return l; +} + +QVariantMap +variantMapFromPyDict( const Calamares::Python::Dictionary& dict ) +{ + QVariantMap m; + for ( const auto item : dict ) + { + m.insert( Calamares::Python::asQString( item.first ), variantFromPyObject( ( item.second ) ) ); + } + return m; +} + +QStringList +stringListFromPyList( const Calamares::Python::List& list ) +{ + QStringList l; + for ( const auto item : list ) + { + l.append( Calamares::Python::asQString( item ) ); + } + return l; +} + +int +raise_on_error( const Calamares::ProcessResult& ec, const QStringList& commandList ) +{ + if ( ec.first == 0 ) + { + return 0; + } + + QString raise = QString( "import subprocess\n" + "e = subprocess.CalledProcessError(%1,\"%2\")\n" ) + .arg( ec.first ) + .arg( commandList.join( ' ' ) ); + if ( !ec.second.isEmpty() ) + { + raise.append( QStringLiteral( "e.output = \"\"\"%1\"\"\"\n" ).arg( ec.second ) ); + } + raise.append( "raise e" ); + py::exec( raise.toStdString() ); + py::error_already_set(); + return ec.first; +} + +int +process_output( Calamares::Utils::RunLocation location, + const QStringList& args, + const Calamares::Python::Object& callback, + const std::string& input, + int timeout ) +{ + Calamares::Utils::Runner r( args ); + r.setLocation( location ); + if ( !callback.is_none() ) + { + if ( py::isinstance< Calamares::Python::List >( callback ) ) + { + QObject::connect( &r, + &decltype( r )::output, + [ list_append = callback.attr( "append" ) ]( const QString& s ) + { list_append( s.toStdString() ); } ); + } + else + { + QObject::connect( + &r, &decltype( r )::output, [ &callback ]( const QString& s ) { callback( s.toStdString() ); } ); + } + r.enableOutputProcessing(); + } + if ( !input.empty() ) + { + r.setInput( QString::fromStdString( input ) ); + } + if ( timeout > 0 ) + { + r.setTimeout( std::chrono::seconds( timeout ) ); + } + + auto result = r.run(); + return raise_on_error( result, args ); +} + +} // namespace + +/** @namespace + * + * This is where the "public Python API" lives. It does not need to + * be a namespace, and it does not need to be public, but it's + * convenient to group things together. + */ +namespace Calamares +{ +namespace Python +{ + +int +target_env_call( const List& args, const std::string& input, int timeout ) +{ + return Calamares::System::instance() + ->targetEnvCommand( + stringListFromPyList( args ), QString(), QString::fromStdString( input ), std::chrono::seconds( timeout ) ) + .first; +} + +int +target_env_call( const std::string& command, const std::string& input, int timeout ) +{ + return Calamares::System::instance() + ->targetEnvCommand( { QString::fromStdString( command ) }, + QString(), + QString::fromStdString( input ), + std::chrono::seconds( timeout ) ) + .first; +} + +int +check_target_env_call( const List& args, const std::string& input, int timeout ) +{ + const auto commandList = stringListFromPyList( args ); + auto ec = Calamares::System::instance()->targetEnvCommand( + commandList, QString(), QString::fromStdString( input ), std::chrono::seconds( timeout ) ); + return raise_on_error( ec, commandList ); +} + +std::string +check_target_env_output( const List& args, const std::string& input, int timeout ) +{ + const auto commandList = stringListFromPyList( args ); + auto ec = Calamares::System::instance()->targetEnvCommand( + commandList, QString(), QString::fromStdString( input ), std::chrono::seconds( timeout ) ); + raise_on_error( ec, commandList ); + return ec.second.toStdString(); +} + +int +target_env_process_output( const List& args, const Object& callback, const std::string& input, int timeout ) +{ + return process_output( + Calamares::System::RunLocation::RunInTarget, stringListFromPyList( args ), callback, input, timeout ); +} +int +host_env_process_output( const List& args, const Object& callback, const std::string& input, int timeout ) +{ + return process_output( + Calamares::System::RunLocation::RunInHost, stringListFromPyList( args ), callback, input, timeout ); +} + +JobProxy::JobProxy( Calamares::Python::Job* parent ) + : prettyName( parent->prettyName().toStdString() ) + , workingPath( parent->workingPath().toStdString() ) + , moduleName( QDir( parent->workingPath() ).dirName().toStdString() ) + , configuration( Calamares::Python::variantMapToPyDict( parent->configuration() ) ) + , m_parent( parent ) +{ +} + +void +JobProxy::setprogress( qreal progress ) +{ + if ( progress >= 0.0 && progress <= 1.0 ) + { + m_parent->emitProgress( progress ); + } +} + + +Calamares::GlobalStorage* GlobalStorageProxy::s_gs_instance = nullptr; + +// The special handling for nullptr is only for the testing +// script for the python bindings, which passes in None; +// normal use will have a GlobalStorage from JobQueue::instance() +// passed in. Testing use will leak the allocated GlobalStorage +// object, but that's OK for testing. +GlobalStorageProxy::GlobalStorageProxy( Calamares::GlobalStorage* gs ) + : m_gs( gs ? gs : s_gs_instance ) +{ + if ( !m_gs ) + { + s_gs_instance = new Calamares::GlobalStorage; + m_gs = s_gs_instance; + } +} + +bool +GlobalStorageProxy::contains( const std::string& key ) const +{ + return m_gs->contains( QString::fromStdString( key ) ); +} + +int +GlobalStorageProxy::count() const +{ + return m_gs->count(); +} + +void +GlobalStorageProxy::insert( const std::string& key, const Object& value ) +{ + m_gs->insert( QString::fromStdString( key ), variantFromPyObject( value ) ); +} + +List +GlobalStorageProxy::keys() const +{ + List pyList; + const auto keys = m_gs->keys(); + for ( const QString& key : keys ) + { + pyList.append( key.toStdString() ); + } + return pyList; +} + +int +GlobalStorageProxy::remove( const std::string& key ) +{ + const QString gsKey( QString::fromStdString( key ) ); + if ( !m_gs->contains( gsKey ) ) + { + cWarning() << "Unknown GS key" << key.c_str(); + } + return m_gs->remove( gsKey ); +} + +Object +GlobalStorageProxy::value( const std::string& key ) const +{ + const QString gsKey( QString::fromStdString( key ) ); + if ( !m_gs->contains( gsKey ) ) + { + cWarning() << "Unknown GS key" << key.c_str(); + return py::none(); + } + return Calamares::Python::variantToPyObject( m_gs->value( gsKey ) ); +} + +} // namespace Python +} // namespace Calamares diff --git a/src/libcalamares/pybind11/Api.h b/src/libcalamares/pybind11/Api.h new file mode 100644 index 0000000000000000000000000000000000000000..27f1db2acce153a841616fd59092f56c50303ed2 --- /dev/null +++ b/src/libcalamares/pybind11/Api.h @@ -0,0 +1,89 @@ +/* === This file is part of Calamares - <https://calamares.io> === + * + * SPDX-FileCopyrightText: 2023 Adriaan de Groot <groot@kde.org> + * SPDX-License-Identifier: GPL-3.0-or-later + * + * Calamares is Free Software: see the License-Identifier above. + * + */ + +#ifndef CALAMARES_PYBIND11_API_H +#define CALAMARES_PYBIND11_API_H + +/** @file + * + * Contains the API that Python modules use from the Python code + * of that module. This is the C++ side that implements the functions + * imported by the Python code as `import libcalamares`. + */ + +#include "PythonTypes.h" + +#include <string> + +namespace Calamares +{ + +class GlobalStorage; +class PythonJob; + +namespace Python __attribute__( ( visibility( "hidden" ) ) ) +{ + int target_env_call( const List& args, const std::string& input, int timeout ); + int target_env_call( const std::string& command, const std::string& input, int timeout ); + int check_target_env_call( const List& args, const std::string& input, int timeout ); + std::string check_target_env_output( const List& args, const std::string& input, int timeout ); + + int target_env_process_output( const List& args, const Object& callback, const std::string& input, int timeout ); + int host_env_process_output( const List& args, const Object& callback, const std::string& input, int timeout ); + + class Job; + + /** @brief Proxy class in Python for the Calamares Job class + * + * This is available as libcalamares.job in Python code. + */ + class JobProxy + { + public: + explicit JobProxy( Calamares::Python::Job* parent ); + + std::string prettyName; + std::string workingPath; + std::string moduleName; + + Dictionary configuration; + + void setprogress( qreal progress ); + + private: + Calamares::Python::Job* m_parent; + }; + + class GlobalStorageProxy + { + public: + explicit GlobalStorageProxy( Calamares::GlobalStorage* gs ); + + bool contains( const std::string& key ) const; + int count() const; + void insert( const std::string& key, const Object& value ); + List keys() const; + int remove( const std::string& key ); + Object value( const std::string& key ) const; + + // This is a helper for scripts that do not go through + // the JobQueue (i.e. the module testpython script), + // which allocate their own (singleton) GlobalStorage. + static Calamares::GlobalStorage* globalStorageInstance() { return s_gs_instance; } + + private: + Calamares::GlobalStorage* m_gs; + static Calamares::GlobalStorage* s_gs_instance; // See globalStorageInstance() + }; + + +} // namespace Python +} // namespace Calamares + +#endif diff --git a/src/libcalamares/pybind11/Pybind11Helpers.h b/src/libcalamares/pybind11/Pybind11Helpers.h new file mode 100644 index 0000000000000000000000000000000000000000..f9a011668d99e3ef90c19ea84a81d01a647cbd3e --- /dev/null +++ b/src/libcalamares/pybind11/Pybind11Helpers.h @@ -0,0 +1,30 @@ +/* === This file is part of Calamares - <https://calamares.io> === + * + * SPDX-FileCopyrightText: 2023 Adriaan de Groot <groot@kde.org> + * SPDX-License-Identifier: GPL-3.0-or-later + * + * Calamares is Free Software: see the License-Identifier above. + * + */ + +#ifndef CALAMARES_PYBIND11_PYBIND11HELPERS_H +#define CALAMARES_PYBIND11_PYBIND11HELPERS_H + +#include "PythonTypes.h" + +#include <QString> + +namespace Calamares +{ +namespace Python __attribute__( ( visibility( "hidden" ) ) ) +{ + inline QString asQString( const pybind11::handle& o ) + { + return QString::fromUtf8( pybind11::str( o ).cast< std::string >().c_str() ); + } + +} // namespace Python +} // namespace Calamares + + +#endif diff --git a/src/libcalamares/python/PythonJob.cpp b/src/libcalamares/pybind11/PythonJob.cpp similarity index 96% rename from src/libcalamares/python/PythonJob.cpp rename to src/libcalamares/pybind11/PythonJob.cpp index cc285d906772c37250318675814c34147d449a2b..69abeb0fd18dd9572508cc76fc2672c48bd944a5 100644 --- a/src/libcalamares/python/PythonJob.cpp +++ b/src/libcalamares/pybind11/PythonJob.cpp @@ -6,14 +6,14 @@ * Calamares is Free Software: see the License-Identifier above. * */ -#include "python/PythonJob.h" +#include "PythonJob.h" #include "CalamaresVersionX.h" #include "GlobalStorage.h" #include "JobQueue.h" +#include "pybind11/Api.h" +#include "pybind11/Pybind11Helpers.h" #include "python/Api.h" -#include "python/Logger.h" -#include "python/Pybind11Helpers.h" #include "utils/Logger.h" #include <QDir> @@ -93,7 +93,13 @@ populate_utils( py::module_& m ) m.def( "load_yaml", &Calamares::Python::load_yaml, "Loads YAML from a file." ); m.def( "target_env_call", - &Calamares::Python::target_env_call, + py::overload_cast<const Calamares::Python::List& , const std::string& , int >(&Calamares::Python::target_env_call), + "Runs command_list in target, returns exit code.", + py::arg( "command_list" ), + py::arg( "input" ) = std::string(), + py::arg( "timeout" ) = 0 ); + m.def( "target_env_call", + py::overload_cast<const std::string& , const std::string& , int >(&Calamares::Python::target_env_call), "Runs command in target, returns exit code.", py::arg( "command_list" ), py::arg( "input" ) = std::string(), diff --git a/src/libcalamares/python/PythonJob.h b/src/libcalamares/pybind11/PythonJob.h similarity index 96% rename from src/libcalamares/python/PythonJob.h rename to src/libcalamares/pybind11/PythonJob.h index abdec676dc4343c382404592659f0a2c0d685dba..197b7a31833d2c52bfca789cb470723175bce736 100644 --- a/src/libcalamares/python/PythonJob.h +++ b/src/libcalamares/pybind11/PythonJob.h @@ -7,8 +7,8 @@ * */ -#ifndef CALAMARES_PYTHON_PYTHONJOB_H -#define CALAMARES_PYTHON_PYTHONJOB_H +#ifndef CALAMARES_PYBIND11_PYTHONJOB_H +#define CALAMARES_PYBIND11_PYTHONJOB_H // This file is called PythonJob.h because it would otherwise // clash with the Job.h in libcalamares proper. diff --git a/src/libcalamares/python/Pybind11Helpers.h b/src/libcalamares/pybind11/PythonTypes.h similarity index 77% rename from src/libcalamares/python/Pybind11Helpers.h rename to src/libcalamares/pybind11/PythonTypes.h index f222f496506ecc7e7a13dff465cf792d87894a8e..ac7a2455377cddab5de06880498d1d78130c5f1e 100644 --- a/src/libcalamares/python/Pybind11Helpers.h +++ b/src/libcalamares/pybind11/PythonTypes.h @@ -1,19 +1,17 @@ /* === This file is part of Calamares - <https://calamares.io> === * - * SPDX-FileCopyrightText: 2023 Adriaan de Groot <groot@kde.org> + * SPDX-FileCopyrightText: 2023, 2024 Adriaan de Groot <groot@kde.org> * SPDX-License-Identifier: GPL-3.0-or-later * * Calamares is Free Software: see the License-Identifier above. * */ -#ifndef CALAMARES_PYTHON_PYBIND11_HELPERS_H -#define CALAMARES_PYTHON_PYBIND11_HELPERS_H +#ifndef CALAMARES_PYBIND11_PYTHONTYPES_H +#define CALAMARES_PYBIND11_PYTHONTYPES_H #include <QString> -#include <string> - QT_WARNING_PUSH QT_WARNING_DISABLE_CLANG( "-Wcovered-switch-default" ) QT_WARNING_DISABLE_CLANG( "-Wfloat-equal" ) @@ -40,19 +38,19 @@ namespace Calamares namespace Python __attribute__( ( visibility( "hidden" ) ) ) { using Dictionary = pybind11::dict; - using String = pybind11::str; using List = pybind11::list; using Object = pybind11::object; - using Float = double; - - inline QString asQString( const pybind11::handle& o ) + inline auto None() { - return QString::fromUtf8( pybind11::str( o ).cast< std::string >().c_str() ); + return pybind11::none(); } + using Integer = pybind11::int_; + using Float = pybind11::float_; + using Boolean = pybind11::bool_; + using String = pybind11::str; } // namespace Python } // namespace Calamares - #endif diff --git a/src/libcalamares/PythonHelper.cpp b/src/libcalamares/pyboost/PythonHelper.cpp similarity index 78% rename from src/libcalamares/PythonHelper.cpp rename to src/libcalamares/pyboost/PythonHelper.cpp index 03cd95e7294c55bccfe5119794a3d19d0e97aafa..cc9df45f12a9138091eb8b2bfbc6147120b60fb1 100644 --- a/src/libcalamares/PythonHelper.cpp +++ b/src/libcalamares/pyboost/PythonHelper.cpp @@ -12,6 +12,7 @@ #include "GlobalStorage.h" #include "compat/Variant.h" +#include "python/Variant.h" #include "utils/Dirs.h" #include "utils/Logger.h" @@ -23,66 +24,6 @@ namespace bp = boost::python; namespace CalamaresPython { -boost::python::object -variantToPyObject( const QVariant& variant ) -{ - QT_WARNING_PUSH - QT_WARNING_DISABLE_CLANG( "-Wswitch-enum" ) - -#if QT_VERSION < QT_VERSION_CHECK( 6, 0, 0 ) - const auto IntVariantType = QVariant::Int; - const auto UIntVariantType = QVariant::UInt; -#else - const auto IntVariantType = QMetaType::Type::Int; - const auto UIntVariantType = QMetaType::Type::UInt; -#endif - // 49 enumeration values not handled - switch ( Calamares::typeOf( variant ) ) - { - case Calamares::MapVariantType: - return variantMapToPyDict( variant.toMap() ); - - case Calamares::HashVariantType: - return variantHashToPyDict( variant.toHash() ); - - case Calamares::ListVariantType: - case Calamares::StringListVariantType: - return variantListToPyList( variant.toList() ); - - case IntVariantType: - return bp::object( variant.toInt() ); - case UIntVariantType: - return bp::object( variant.toUInt() ); - - case Calamares::LongLongVariantType: - return bp::object( variant.toLongLong() ); - case Calamares::ULongLongVariantType: - return bp::object( variant.toULongLong() ); - - case Calamares::DoubleVariantType: - return bp::object( variant.toDouble() ); - - case Calamares::CharVariantType: -#if QT_VERSION < QT_VERSION_CHECK( 6, 0, 0 ) -#else - // In Qt6, QChar is also available and different from CharVariantType - case QMetaType::Type::QChar: -#endif - case Calamares::StringVariantType: - return bp::object( variant.toString().toStdString() ); - - case Calamares::BoolVariantType: - return bp::object( variant.toBool() ); - -#if QT_VERSION < QT_VERSION_CHECK( 6, 0, 0 ) - case QVariant::Invalid: -#endif - default: - return bp::object(); - } - QT_WARNING_POP -} - QVariant variantFromPyObject( const boost::python::object& pyObject ) { @@ -123,17 +64,6 @@ variantFromPyObject( const boost::python::object& pyObject ) } } -boost::python::list -variantListToPyList( const QVariantList& variantList ) -{ - bp::list pyList; - for ( const QVariant& variant : variantList ) - { - pyList.append( variantToPyObject( variant ) ); - } - return pyList; -} - QVariantList variantListFromPyList( const boost::python::list& pyList ) { @@ -145,17 +75,6 @@ variantListFromPyList( const boost::python::list& pyList ) return list; } -boost::python::dict -variantMapToPyDict( const QVariantMap& variantMap ) -{ - bp::dict pyDict; - for ( auto it = variantMap.constBegin(); it != variantMap.constEnd(); ++it ) - { - pyDict[ it.key().toStdString() ] = variantToPyObject( it.value() ); - } - return pyDict; -} - QVariantMap variantMapFromPyDict( const boost::python::dict& pyDict ) { @@ -179,17 +98,6 @@ variantMapFromPyDict( const boost::python::dict& pyDict ) return map; } -boost::python::dict -variantHashToPyDict( const QVariantHash& variantHash ) -{ - bp::dict pyDict; - for ( auto it = variantHash.constBegin(); it != variantHash.constEnd(); ++it ) - { - pyDict[ it.key().toStdString() ] = variantToPyObject( it.value() ); - } - return pyDict; -} - QVariantHash variantHashFromPyDict( const boost::python::dict& pyDict ) { @@ -460,7 +368,7 @@ GlobalStoragePythonWrapper::value( const std::string& key ) const { cWarning() << "Unknown GS key" << key.c_str(); } - return CalamaresPython::variantToPyObject( m_gs->value( gsKey ) ); + return Calamares::Python::variantToPyObject( m_gs->value( gsKey ) ); } } // namespace CalamaresPython diff --git a/src/libcalamares/PythonHelper.h b/src/libcalamares/pyboost/PythonHelper.h similarity index 82% rename from src/libcalamares/PythonHelper.h rename to src/libcalamares/pyboost/PythonHelper.h index f307894f0225dcd21675554af1a732e4760c61a5..e3b27e6d59abd1698cba3a26155888a668e9ae35 100644 --- a/src/libcalamares/PythonHelper.h +++ b/src/libcalamares/pyboost/PythonHelper.h @@ -8,12 +8,12 @@ * */ -#ifndef CALAMARES_PYTHONJOBHELPER_H -#define CALAMARES_PYTHONJOBHELPER_H +#ifndef CALAMARES_PYBOOST_PYTHONHELPER_H +#define CALAMARES_PYBOOST_PYTHONHELPER_H #include "DllMacro.h" #include "PythonJob.h" -#include "utils/BoostPython.h" +#include "PythonTypes.h" #include <QStringList> @@ -25,16 +25,9 @@ class GlobalStorage; namespace CalamaresPython { -DLLEXPORT boost::python::object variantToPyObject( const QVariant& variant ); DLLEXPORT QVariant variantFromPyObject( const boost::python::object& pyObject ); - -DLLEXPORT boost::python::list variantListToPyList( const QVariantList& variantList ); DLLEXPORT QVariantList variantListFromPyList( const boost::python::list& pyList ); - -DLLEXPORT boost::python::dict variantMapToPyDict( const QVariantMap& variantMap ); DLLEXPORT QVariantMap variantMapFromPyDict( const boost::python::dict& pyDict ); - -DLLEXPORT boost::python::dict variantHashToPyDict( const QVariantHash& variantHash ); DLLEXPORT QVariantHash variantHashFromPyDict( const boost::python::dict& pyDict ); diff --git a/src/libcalamares/PythonJob.cpp b/src/libcalamares/pyboost/PythonJob.cpp similarity index 89% rename from src/libcalamares/PythonJob.cpp rename to src/libcalamares/pyboost/PythonJob.cpp index 6f5ca19277b5472a85f5af68da13a367164c85bc..c3051cd7802e4bad87a56445a1d34eb92e0582d8 100644 --- a/src/libcalamares/PythonJob.cpp +++ b/src/libcalamares/pyboost/PythonJob.cpp @@ -9,12 +9,14 @@ */ #include "PythonJob.h" +#include "PythonHelper.h" +#include "PythonJobApi.h" +#include "PythonTypes.h" + #include "CalamaresVersion.h" #include "GlobalStorage.h" #include "JobQueue.h" -#include "PythonHelper.h" -#include "PythonJobApi.h" -#include "utils/BoostPython.h" +#include "python/Api.h" #include "utils/Logger.h" #include <QDir> @@ -31,7 +33,7 @@ namespace bp = boost::python; QT_WARNING_PUSH QT_WARNING_DISABLE_CLANG( "-Wdisabled-macro-expansion" ) -BOOST_PYTHON_FUNCTION_OVERLOADS( mount_overloads, CalamaresPython::mount, 2, 4 ); +BOOST_PYTHON_FUNCTION_OVERLOADS( mount_overloads, Calamares::Python::mount, 2, 4 ); BOOST_PYTHON_FUNCTION_OVERLOADS( target_env_call_str_overloads, CalamaresPython::target_env_call, 1, 3 ); BOOST_PYTHON_FUNCTION_OVERLOADS( target_env_call_list_overloads, CalamaresPython::target_env_call, 1, 3 ); BOOST_PYTHON_FUNCTION_OVERLOADS( check_target_env_call_str_overloads, CalamaresPython::check_target_env_call, 1, 3 ); @@ -91,25 +93,25 @@ BOOST_PYTHON_MODULE( libcalamares ) // .. Logging functions bp::def( - "debug", &CalamaresPython::debug, bp::args( "s" ), "Writes the given string to the Calamares debug stream." ); + "debug", &Calamares::Python::debug, bp::args( "s" ), "Writes the given string to the Calamares debug stream." ); bp::def( "warning", - &CalamaresPython::warning, + &Calamares::Python::warning, bp::args( "s" ), "Writes the given string to the Calamares warning stream." ); bp::def( "warn", - &CalamaresPython::warning, + &Calamares::Python::warning, bp::args( "s" ), "Writes the given string to the Calamares warning stream." ); bp::def( - "error", &CalamaresPython::error, bp::args( "s" ), "Writes the given string to the Calamares error stream." ); + "error", &Calamares::Python::error, bp::args( "s" ), "Writes the given string to the Calamares error stream." ); // .. YAML functions - bp::def( "load_yaml", &CalamaresPython::load_yaml, bp::args( "path" ), "Loads YAML from a file." ); + bp::def( "load_yaml", &Calamares::Python::load_yaml, bp::args( "path" ), "Loads YAML from a file." ); // .. Filesystem functions bp::def( "mount", - &CalamaresPython::mount, + &Calamares::Python::mount, mount_overloads( bp::args( "device_path", "mount_point", "filesystem_name", "options" ), "Runs the mount utility with the specified parameters.\n" "Returns the program's exit code, or:\n" @@ -130,8 +132,8 @@ BOOST_PYTHON_MODULE( libcalamares ) "-4 = QProcess timeout" ) ); bp::def( "target_env_call", static_cast< int ( * )( const bp::list&, const std::string&, int ) >( &CalamaresPython::target_env_call ), - target_env_call_list_overloads( bp::args( "args", "stdin", "timeout" ), - "Runs the specified command in the chroot of the target system.\n" + target_env_call_list_overloads( bp::args( "command_list", "stdin", "timeout" ), + "Runs the specified command_list in the chroot of the target system.\n" "Returns the program's exit code, or:\n" "-1 = QProcess crash\n" "-2 = QProcess cannot start\n" @@ -178,7 +180,7 @@ BOOST_PYTHON_MODULE( libcalamares ) // .. String functions bp::def( "obscure", - &CalamaresPython::obscure, + &Calamares::Python::obscure, bp::args( "s" ), "Simple string obfuscation function based on KStringHandler::obscure.\n" "Returns a string, generated using a simple symmetric encryption.\n" @@ -187,10 +189,10 @@ BOOST_PYTHON_MODULE( libcalamares ) // .. Translation functions bp::def( "gettext_languages", - &CalamaresPython::gettext_languages, + &Calamares::Python::gettext_languages, "Returns list of languages (most to least-specific) for gettext." ); - bp::def( "gettext_path", &CalamaresPython::gettext_path, "Returns path for gettext search." ); + bp::def( "gettext_path", &Calamares::Python::gettext_path, "Returns path for gettext search." ); } @@ -210,7 +212,6 @@ PythonJob::PythonJob( const QString& scriptFile, , m_d( std::make_unique< Private >() ) , m_scriptFile( scriptFile ) , m_workingPath( workingPath ) - , m_description() , m_configurationMap( moduleConfiguration ) { } @@ -229,13 +230,14 @@ QString PythonJob::prettyStatusMessage() const { // The description is updated when progress is reported, see emitProgress() - if ( m_description.isEmpty() ) + const auto s = getDescription(); + if ( s.isEmpty() ) { return tr( "Running %1 operation…", "@status" ).arg( QDir( m_workingPath ).dirName() ); } else { - return m_description; + return s; } } @@ -296,26 +298,27 @@ PythonJob::exec() bp::object entryPoint = scriptNamespace[ "run" ]; m_d->m_prettyStatusMessage = scriptNamespace.get( "pretty_status_message", bp::object() ); - m_description = pythonStringMethod( scriptNamespace, "pretty_name" ); - if ( m_description.isEmpty() ) + QString possibleDescription = pythonStringMethod( scriptNamespace, "pretty_name" ); + if ( possibleDescription.isEmpty() ) { bp::extract< std::string > entryPoint_doc_attr( entryPoint.attr( "__doc__" ) ); if ( entryPoint_doc_attr.check() ) { - m_description = QString::fromStdString( entryPoint_doc_attr() ).trimmed(); - auto i_newline = m_description.indexOf( '\n' ); + possibleDescription= QString::fromStdString( entryPoint_doc_attr() ).trimmed(); + auto i_newline = possibleDescription.indexOf( '\n' ); if ( i_newline > 0 ) { - m_description.truncate( i_newline ); + possibleDescription.truncate( i_newline ); } - cDebug() << Logger::SubEntry << "Job description from __doc__" << prettyName() << '=' << m_description; + cDebug() << Logger::SubEntry << "Job description from __doc__" << prettyName() << '=' << possibleDescription; } } else { - cDebug() << Logger::SubEntry << "Job description from pretty_name" << prettyName() << '=' << m_description; + cDebug() << Logger::SubEntry << "Job description from pretty_name" << prettyName() << '=' << possibleDescription; } + setDescription( possibleDescription); emit progress( 0 ); bp::object runResult = entryPoint(); @@ -362,7 +365,7 @@ PythonJob::emitProgress( qreal progressValue ) r = result.check() ? QString::fromStdString( result() ).trimmed() : QString(); if ( !r.isEmpty() ) { - m_description = r; + setDescription(r); } } emit progress( progressValue ); @@ -376,4 +379,16 @@ PythonJob::setInjectedPreScript( const char* preScript ) << ( preScript ? strlen( preScript ) : 0 ); } +QString PythonJob::getDescription() const +{ + QMutexLocker l(&m_descriptionMutex); + return m_description; +} + +void PythonJob::setDescription(const QString & s) +{ + QMutexLocker l(&m_descriptionMutex); + m_description = s; +} + } // namespace Calamares diff --git a/src/libcalamares/PythonJob.h b/src/libcalamares/pyboost/PythonJob.h similarity index 86% rename from src/libcalamares/PythonJob.h rename to src/libcalamares/pyboost/PythonJob.h index 7a86b8a350f7135cf906b161b147009c812e5652..6a7a4f5fa3a8599ba11b09a301df646d0f3f7787 100644 --- a/src/libcalamares/PythonJob.h +++ b/src/libcalamares/pyboost/PythonJob.h @@ -8,13 +8,14 @@ * */ -#ifndef CALAMARES_PYTHONJOB_H -#define CALAMARES_PYTHONJOB_H +#ifndef CALAMARES_PYBOOST_PYTHONJOB_H +#define CALAMARES_PYBOOST_PYTHONJOB_H #include "DllMacro.h" #include "Job.h" #include "modulesystem/InstanceKey.h" +#include <QMutex> #include <QVariantMap> #include <memory> @@ -69,8 +70,13 @@ private: std::unique_ptr< Private > m_d; QString m_scriptFile; QString m_workingPath; - QString m_description; QVariantMap m_configurationMap; + + mutable QMutex m_descriptionMutex; // Guards access to m_description, because that is read and written from multiple threads + QString m_description; + + QString getDescription() const; + void setDescription(const QString & s); }; } // namespace Calamares diff --git a/src/libcalamares/PythonJobApi.cpp b/src/libcalamares/pyboost/PythonJobApi.cpp similarity index 56% rename from src/libcalamares/PythonJobApi.cpp rename to src/libcalamares/pyboost/PythonJobApi.cpp index ca1e0bc92529893fce0ad73576cc9ef5344dd147..64fe7dd828547fadc561917016e9c065c87a41f5 100644 --- a/src/libcalamares/PythonJobApi.cpp +++ b/src/libcalamares/pyboost/PythonJobApi.cpp @@ -10,11 +10,13 @@ #include "PythonJobApi.h" +#include "PythonHelper.h" + #include "GlobalStorage.h" #include "JobQueue.h" -#include "PythonHelper.h" #include "locale/Global.h" -#include "partition/Mount.h" +#include "python/Api.h" +#include "python/Variant.h" #include "utils/Logger.h" #include "utils/RAII.h" #include "utils/Runner.h" @@ -73,18 +75,6 @@ target_env_command( const QStringList& args, const std::string& input, int timeo namespace CalamaresPython { -int -mount( const std::string& device_path, - const std::string& mount_point, - const std::string& filesystem_name, - const std::string& options ) -{ - return Calamares::Partition::mount( QString::fromStdString( device_path ), - QString::fromStdString( mount_point ), - QString::fromStdString( filesystem_name ), - QString::fromStdString( options ) ); -} - int target_env_call( const std::string& command, const std::string& input, int timeout ) { @@ -134,44 +124,6 @@ check_target_env_output( const bp::list& args, const std::string& input, int tim return ec.second.toStdString(); } -static const char output_prefix[] = "[PYTHON JOB]:"; -static inline void -log_action( unsigned int level, const std::string& s ) -{ - Logger::CDebug( level ) << output_prefix << QString::fromStdString( s ); -} - -void -debug( const std::string& s ) -{ - log_action( Logger::LOGDEBUG, s ); -} - -void -warning( const std::string& s ) -{ - log_action( Logger::LOGWARNING, s ); -} - -void -error( const std::string& s ) -{ - log_action( Logger::LOGERROR, s ); -} - -boost::python::dict -load_yaml( const std::string& path ) -{ - const QString filePath = QString::fromStdString( path ); - bool ok = false; - auto map = Calamares::YAML::load( filePath, &ok ); - if ( !ok ) - { - cWarning() << "Loading YAML from" << filePath << "failed."; - } - return variantMapToPyDict( map ); -} - PythonJobInterface::PythonJobInterface( Calamares::PythonJob* parent ) : m_parent( parent ) { @@ -179,7 +131,7 @@ PythonJobInterface::PythonJobInterface( Calamares::PythonJob* parent ) moduleName = moduleDir.dirName().toStdString(); prettyName = m_parent->prettyName().toStdString(); workingPath = m_parent->m_workingPath.toStdString(); - configuration = CalamaresPython::variantMapToPyDict( m_parent->m_configurationMap ); + configuration = Calamares::Python::variantMapToPyDict( m_parent->m_configurationMap ); } void @@ -252,113 +204,4 @@ host_env_process_output( const boost::python::list& args, return _process_output( Calamares::Utils::RunLocation::RunInHost, args, callback, input, timeout ); } -std::string -obscure( const std::string& string ) -{ - return Calamares::String::obscure( QString::fromStdString( string ) ).toStdString(); -} - -static QStringList -_gettext_languages() -{ - QStringList languages; - - // There are two ways that Python jobs can be initialised: - // - through JobQueue, in which case that has an instance which holds - // a GlobalStorage object, or - // - through the Python test-script, which initialises its - // own GlobalStoragePythonWrapper, which then holds a - // GlobalStorage object for all of Python. - Calamares::JobQueue* jq = Calamares::JobQueue::instance(); - Calamares::GlobalStorage* gs - = jq ? jq->globalStorage() : CalamaresPython::GlobalStoragePythonWrapper::globalStorageInstance(); - - QString lang = Calamares::Locale::readGS( *gs, QStringLiteral( "LANG" ) ); - if ( !lang.isEmpty() ) - { - languages.append( lang ); - if ( lang.indexOf( '.' ) > 0 ) - { - lang.truncate( lang.indexOf( '.' ) ); - languages.append( lang ); - } - if ( lang.indexOf( '_' ) > 0 ) - { - lang.truncate( lang.indexOf( '_' ) ); - languages.append( lang ); - } - } - return languages; -} - -bp::list -gettext_languages() -{ - bp::list pyList; - for ( auto lang : _gettext_languages() ) - { - pyList.append( lang.toStdString() ); - } - return pyList; -} - -static void -_add_localedirs( QStringList& pathList, const QString& candidate ) -{ - if ( !candidate.isEmpty() && !pathList.contains( candidate ) ) - { - pathList.prepend( candidate ); - if ( QDir( candidate ).cd( "lang" ) ) - { - pathList.prepend( candidate + "/lang" ); - } - } -} - -bp::object -gettext_path() -{ - // Going to log informatively just once - static bool first_time = true; - cScopedAssignment( &first_time, false ); - - // TODO: distinguish between -d runs and normal runs - // TODO: can we detect DESTDIR-installs? - QStringList candidatePaths - = QStandardPaths::locateAll( QStandardPaths::GenericDataLocation, "locale", QStandardPaths::LocateDirectory ); - QString extra = QCoreApplication::applicationDirPath(); - _add_localedirs( candidatePaths, extra ); // Often /usr/local/bin - if ( !extra.isEmpty() ) - { - QDir d( extra ); - if ( d.cd( "../share/locale" ) ) // Often /usr/local/bin/../share/locale -> /usr/local/share/locale - { - _add_localedirs( candidatePaths, d.canonicalPath() ); - } - } - _add_localedirs( candidatePaths, QDir().canonicalPath() ); // . - - if ( first_time ) - { - cDebug() << "Determining gettext path from" << candidatePaths; - } - - QStringList candidateLanguages = _gettext_languages(); - for ( const auto& lang : candidateLanguages ) - { - for ( auto localedir : candidatePaths ) - { - QDir ldir( localedir ); - if ( ldir.cd( lang ) ) - { - Logger::CDebug( Logger::LOGDEBUG ) - << output_prefix << "Found gettext" << lang << "in" << ldir.canonicalPath(); - return bp::object( localedir.toStdString() ); - } - } - } - cWarning() << "No translation found for languages" << candidateLanguages; - return bp::object(); // None -} - } // namespace CalamaresPython diff --git a/src/libcalamares/PythonJobApi.h b/src/libcalamares/pyboost/PythonJobApi.h similarity index 76% rename from src/libcalamares/PythonJobApi.h rename to src/libcalamares/pyboost/PythonJobApi.h index 373812187908935770d314e2e9bc1ae5dc308462..d7f9e5156e251ffc57f80359ed17cf96a051a099 100644 --- a/src/libcalamares/PythonJobApi.h +++ b/src/libcalamares/pyboost/PythonJobApi.h @@ -8,13 +8,15 @@ * */ -#ifndef PYTHONJOBAPI_H -#define PYTHONJOBAPI_H +#ifndef CALAMARES_PYBOOST_PYTHONJOBAPI_H +#define CALAMARES_PYBOOST_PYTHONJOBAPI_H -#include "utils/BoostPython.h" +#include "PythonTypes.h" #include <qglobal.h> // For qreal +#include <string> + namespace Calamares { class PythonJob; @@ -23,11 +25,6 @@ class PythonJob; namespace CalamaresPython { -int mount( const std::string& device_path, - const std::string& mount_point, - const std::string& filesystem_name = std::string(), - const std::string& options = std::string() ); - int target_env_call( const std::string& command, const std::string& input = std::string(), int timeout = 0 ); int target_env_call( const boost::python::list& args, const std::string& input = std::string(), int timeout = 0 ); @@ -52,21 +49,6 @@ int host_env_process_output( const boost::python::list& args, const std::string& input = std::string(), int timeout = 0 ); -std::string obscure( const std::string& string ); - -boost::python::object gettext_path(); - -boost::python::list gettext_languages(); - -void debug( const std::string& s ); -void warning( const std::string& s ); -void error( const std::string& s ); - -/** @brief Loads YAML and returns (nested) dicts representing it - * - */ -boost::python::dict load_yaml( const std::string& path ); - class PythonJobInterface { public: diff --git a/src/libcalamares/utils/BoostPython.h b/src/libcalamares/pyboost/PythonTypes.h similarity index 77% rename from src/libcalamares/utils/BoostPython.h rename to src/libcalamares/pyboost/PythonTypes.h index 5cc5c31760309393d60c83a63757c135c29c9fb1..d745255d7fd201fa7abbf336d6668d9cfbd3a828 100644 --- a/src/libcalamares/utils/BoostPython.h +++ b/src/libcalamares/pyboost/PythonTypes.h @@ -1,6 +1,6 @@ /* === This file is part of Calamares - <https://calamares.io> === * - * SPDX-FileCopyrightText: 2019-2020 Adriaan de Groot <groot@kde.org> + * SPDX-FileCopyrightText: 2019-2020, 2024 Adriaan de Groot <groot@kde.org> * SPDX-License-Identifier: GPL-3.0-or-later * * Calamares is Free Software: see the License-Identifier above. @@ -16,8 +16,8 @@ * This convenience header handles including all the bits we need for * Python support, while silencing warnings. */ -#ifndef UTILS_BOOSTPYTHON_H -#define UTILS_BOOSTPYTHON_H +#ifndef CALAMARES_PYBOOST_PYTHONTYPES_H +#define CALAMARES_PYBOOST_PYTHONTYPES_H #include <qglobal.h> @@ -59,4 +59,24 @@ QT_WARNING_DISABLE_CLANG( "-Wreserved-id-macro" ) QT_WARNING_POP +namespace Calamares +{ +namespace Python __attribute__( ( visibility( "hidden" ) ) ) +{ + using Dictionary = boost::python::dict; + using List = boost::python::list; + using Object = boost::python::object; + + inline auto None() + { + return Object(); + } + + using Integer = Object; + using Float = Object; + using Boolean = Object; + using String = Object; +} // namespace Python +} // namespace Calamares + #endif diff --git a/src/libcalamares/python/Api.cpp b/src/libcalamares/python/Api.cpp index 602389cf0ba6d534ceb512e3dc2347eba2bf9653..d7c25e4a424d98f5448bad3c047f3a55df62d0f9 100644 --- a/src/libcalamares/python/Api.cpp +++ b/src/libcalamares/python/Api.cpp @@ -1,221 +1,51 @@ /* === This file is part of Calamares - <https://calamares.io> === * * SPDX-FileCopyrightText: 2014 Teo Mrnjavac <teo@kde.org> - * SPDX-FileCopyrightText: 2017-2020, 2023 Adriaan de Groot <groot@kde.org> + * SPDX-FileCopyrightText: 2017-2020, 2023-2024 Adriaan de Groot <groot@kde.org> * SPDX-License-Identifier: GPL-3.0-or-later * * Calamares is Free Software: see the License-Identifier above. * */ -#include "python/Api.h" +#include "Api.h" + +#include "Variant.h" #include "GlobalStorage.h" #include "JobQueue.h" -#include "compat/Variant.h" #include "locale/Global.h" #include "partition/Mount.h" -#include "python/Pybind11Helpers.h" -#include "python/PythonJob.h" #include "utils/Logger.h" #include "utils/RAII.h" -#include "utils/Runner.h" #include "utils/String.h" -#include "utils/System.h" #include "utils/Yaml.h" #include <QCoreApplication> #include <QDir> #include <QStandardPaths> -namespace py = pybind11; - -/** @namespace - * - * Helper functions for converting Python (pybind11) types to Qt types. - */ namespace { -// Forward declarations, since most of these are mutually recursive -Calamares::Python::List variantListToPyList( const QVariantList& variantList ); -Calamares::Python::Dictionary variantMapToPyDict( const QVariantMap& variantMap ); -Calamares::Python::Dictionary variantHashToPyDict( const QVariantHash& variantHash ); - -py::object -variantToPyObject( const QVariant& variant ) -{ - QT_WARNING_PUSH - QT_WARNING_DISABLE_CLANG( "-Wswitch-enum" ) - -#if QT_VERSION < QT_VERSION_CHECK( 6, 0, 0 ) - const auto IntVariantType = QVariant::Int; - const auto UIntVariantType = QVariant::UInt; -#else - const auto IntVariantType = QMetaType::Type::Int; - const auto UIntVariantType = QMetaType::Type::UInt; -#endif - // 49 enumeration values not handled - switch ( Calamares::typeOf( variant ) ) - { - case Calamares::MapVariantType: - return variantMapToPyDict( variant.toMap() ); - - case Calamares::HashVariantType: - return variantHashToPyDict( variant.toHash() ); - - case Calamares::ListVariantType: - case Calamares::StringListVariantType: - return variantListToPyList( variant.toList() ); - case IntVariantType: - return py::int_( variant.toInt() ); - case UIntVariantType: - return py::int_( variant.toUInt() ); +///@brief Prefix added to Python log-messages +constexpr char output_prefix[] = "[PYTHON JOB]:"; - case Calamares::LongLongVariantType: - return py::int_( variant.toLongLong() ); - case Calamares::ULongLongVariantType: - return py::int_( variant.toULongLong() ); - - case Calamares::DoubleVariantType: - return py::float_( variant.toDouble() ); - - case Calamares::CharVariantType: -#if QT_VERSION < QT_VERSION_CHECK( 6, 0, 0 ) -#else - // In Qt6, QChar is also available and different from CharVariantType - case QMetaType::Type::QChar: -#endif - case Calamares::StringVariantType: - return Calamares::Python::String( variant.toString().toStdString() ); - - case Calamares::BoolVariantType: - return py::bool_( variant.toBool() ); - -#if QT_VERSION < QT_VERSION_CHECK( 6, 0, 0 ) - case QVariant::Invalid: -#endif - default: - return py::none(); - } - QT_WARNING_POP -} - -Calamares::Python::List -variantListToPyList( const QVariantList& variantList ) -{ - Calamares::Python::List pyList; - for ( const QVariant& variant : variantList ) - { - pyList.append( variantToPyObject( variant ) ); - } - return pyList; -} - -Calamares::Python::Dictionary -variantMapToPyDict( const QVariantMap& variantMap ) -{ - Calamares::Python::Dictionary pyDict; - for ( auto it = variantMap.constBegin(); it != variantMap.constEnd(); ++it ) - { - pyDict[ Calamares::Python::String( it.key().toStdString() ) ] = variantToPyObject( it.value() ); - } - return pyDict; -} - -Calamares::Python::Dictionary -variantHashToPyDict( const QVariantHash& variantHash ) -{ - Calamares::Python::Dictionary pyDict; - for ( auto it = variantHash.constBegin(); it != variantHash.constEnd(); ++it ) - { - pyDict[ Calamares::Python::String( it.key().toStdString() ) ] = variantToPyObject( it.value() ); - } - return pyDict; -} - -QVariantList variantListFromPyList( const Calamares::Python::List& list ); -QVariantMap variantMapFromPyDict( const Calamares::Python::Dictionary& dict ); - -QVariant -variantFromPyObject( const py::handle& o ) -{ - if ( py::isinstance< Calamares::Python::Dictionary >( o ) ) - { - return variantMapFromPyDict( py::cast< Calamares::Python::Dictionary >( o ) ); - } - else if ( py::isinstance< Calamares::Python::List >( o ) ) - { - return variantListFromPyList( py::cast< Calamares::Python::List >( o ) ); - } - else if ( py::isinstance< py::int_ >( o ) ) - { - return QVariant( qlonglong( py::cast< py::int_ >( o ) ) ); - } - else if ( py::isinstance< py::float_ >( o ) ) - { - return QVariant( double( py::cast< py::float_ >( o ) ) ); - } - else if ( py::isinstance< py::str >( o ) ) - { - return QVariant( QString::fromStdString( std::string( py::str( o ) ) ) ); - } - else if ( py::isinstance< py::bool_ >( o ) ) - { - return QVariant( bool( py::cast< py::bool_ >( o ) ) ); - } - - return QVariant(); -} - -QVariantList -variantListFromPyList( const Calamares::Python::List& list ) -{ - QVariantList l; - for ( const auto item : list ) - { - l.append( variantFromPyObject( item ) ); - } - return l; -} - -QVariantMap -variantMapFromPyDict( const Calamares::Python::Dictionary& dict ) -{ - QVariantMap m; - for ( const auto item : dict ) - { - m.insert( Calamares::Python::asQString( item.first ), variantFromPyObject( ( item.second ) ) ); - } - return m; -} - -QStringList -stringListFromPyList( const Calamares::Python::List& list ) -{ - QStringList l; - for ( const auto item : list ) - { - l.append( Calamares::Python::asQString( item ) ); - } - return l; -} - -const char output_prefix[] = "[PYTHON JOB]:"; +///@brief Helper function to log a message (adds prefix, wrangles types) inline void log_action( unsigned int level, const std::string& s ) { Logger::CDebug( level ) << output_prefix << QString::fromStdString( s ); } -static Calamares::GlobalStorage* -_global_storage() +Calamares::GlobalStorage* +own_global_storage() { static Calamares::GlobalStorage* p = new Calamares::GlobalStorage; return p; } -static QStringList -_gettext_languages() +QStringList +languages_from_global_storage() { QStringList languages; @@ -226,7 +56,7 @@ _gettext_languages() // own GlobalStorageProxy, which then holds a // GlobalStorage object for all of Python. Calamares::JobQueue* jq = Calamares::JobQueue::instance(); - Calamares::GlobalStorage* gs = jq ? jq->globalStorage() : _global_storage(); + Calamares::GlobalStorage* gs = jq ? jq->globalStorage() : own_global_storage(); QString lang = Calamares::Locale::readGS( *gs, QStringLiteral( "LANG" ) ); if ( !lang.isEmpty() ) @@ -246,8 +76,8 @@ _gettext_languages() return languages; } -static void -_add_localedirs( QStringList& pathList, const QString& candidate ) +void +append_language_directory( QStringList& pathList, const QString& candidate ) { if ( !candidate.isEmpty() && !pathList.contains( candidate ) ) { @@ -259,74 +89,8 @@ _add_localedirs( QStringList& pathList, const QString& candidate ) } } -int -raise_on_error( const Calamares::ProcessResult& ec, const QStringList& commandList ) -{ - if ( ec.first == 0 ) - { - return 0; - } - - QString raise = QString( "import subprocess\n" - "e = subprocess.CalledProcessError(%1,\"%2\")\n" ) - .arg( ec.first ) - .arg( commandList.join( ' ' ) ); - if ( !ec.second.isEmpty() ) - { - raise.append( QStringLiteral( "e.output = \"\"\"%1\"\"\"\n" ).arg( ec.second ) ); - } - raise.append( "raise e" ); - py::exec( raise.toStdString() ); - py::error_already_set(); - return ec.first; -} - -int -process_output( Calamares::Utils::RunLocation location, - const QStringList& args, - const Calamares::Python::Object& callback, - const std::string& input, - int timeout ) -{ - Calamares::Utils::Runner r( args ); - r.setLocation( location ); - if ( !callback.is_none() ) - { - if ( py::isinstance< Calamares::Python::List >( callback ) ) - { - QObject::connect( &r, - &decltype( r )::output, - [ list_append = callback.attr( "append" ) ]( const QString& s ) - { list_append( s.toStdString() ); } ); - } - else - { - QObject::connect( - &r, &decltype( r )::output, [ &callback ]( const QString& s ) { callback( s.toStdString() ); } ); - } - r.enableOutputProcessing(); - } - if ( !input.empty() ) - { - r.setInput( QString::fromStdString( input ) ); - } - if ( timeout > 0 ) - { - r.setTimeout( std::chrono::seconds( timeout ) ); - } - - auto result = r.run(); - return raise_on_error( result, args ); } -} // namespace - -/** @namespace - * - * This is where the "public Python API" lives. It does not need to - * be a namespace, and it does not need to be public, but it's - * convenient to group things together. - */ namespace Calamares { namespace Python @@ -367,21 +131,21 @@ load_yaml( const std::string& path ) cWarning() << "Loading YAML from" << filePath << "failed."; } - return variantMapToPyDict( map ); + return Calamares::Python::variantMapToPyDict( map ); } -py::list +Python::List gettext_languages() { - py::list pyList; - for ( auto lang : _gettext_languages() ) + Python::List pyList; + for ( const auto & lang : languages_from_global_storage() ) { pyList.append( lang.toStdString() ); } return pyList; } -py::object +Python::Object gettext_path() { // Going to log informatively just once @@ -393,26 +157,26 @@ gettext_path() QStringList candidatePaths = QStandardPaths::locateAll( QStandardPaths::GenericDataLocation, "locale", QStandardPaths::LocateDirectory ); QString extra = QCoreApplication::applicationDirPath(); - _add_localedirs( candidatePaths, extra ); // Often /usr/local/bin + append_language_directory( candidatePaths, extra ); // Often /usr/local/bin if ( !extra.isEmpty() ) { QDir d( extra ); if ( d.cd( "../share/locale" ) ) // Often /usr/local/bin/../share/locale -> /usr/local/share/locale { - _add_localedirs( candidatePaths, d.canonicalPath() ); + append_language_directory( candidatePaths, d.canonicalPath() ); } } - _add_localedirs( candidatePaths, QDir().canonicalPath() ); // . + append_language_directory( candidatePaths, QDir().canonicalPath() ); // Current directory, e.g. "." if ( first_time ) { cDebug() << "Determining gettext path from" << candidatePaths; } - QStringList candidateLanguages = _gettext_languages(); + QStringList candidateLanguages = languages_from_global_storage(); for ( const auto& lang : candidateLanguages ) { - for ( auto localedir : candidatePaths ) + for ( const auto & localedir : candidatePaths ) { QDir ldir( localedir ); if ( ldir.cd( lang ) ) @@ -424,48 +188,7 @@ gettext_path() } } cWarning() << "No translation found for languages" << candidateLanguages; - return py::none(); // None -} - -int -target_env_call( const List& args, const std::string& input, int timeout ) -{ - return Calamares::System::instance() - ->targetEnvCommand( - stringListFromPyList( args ), QString(), QString::fromStdString( input ), std::chrono::seconds( timeout ) ) - .first; -} - -int -check_target_env_call( const List& args, const std::string& input, int timeout ) -{ - const auto commandList = stringListFromPyList( args ); - auto ec = Calamares::System::instance()->targetEnvCommand( - commandList, QString(), QString::fromStdString( input ), std::chrono::seconds( timeout ) ); - return raise_on_error( ec, commandList ); -} - -std::string -check_target_env_output( const List& args, const std::string& input, int timeout ) -{ - const auto commandList = stringListFromPyList( args ); - auto ec = Calamares::System::instance()->targetEnvCommand( - commandList, QString(), QString::fromStdString( input ), std::chrono::seconds( timeout ) ); - raise_on_error( ec, commandList ); - return ec.second.toStdString(); -} - -int -target_env_process_output( const List& args, const Object& callback, const std::string& input, int timeout ) -{ - return process_output( - Calamares::System::RunLocation::RunInTarget, stringListFromPyList( args ), callback, input, timeout ); -} -int -host_env_process_output( const List& args, const Object& callback, const std::string& input, int timeout ) -{ - return process_output( - Calamares::System::RunLocation::RunInHost, stringListFromPyList( args ), callback, input, timeout ); + return Python::None(); } int @@ -480,94 +203,5 @@ mount( const std::string& device_path, QString::fromStdString( options ) ); } -JobProxy::JobProxy( Calamares::Python::Job* parent ) - : prettyName( parent->prettyName().toStdString() ) - , workingPath( parent->workingPath().toStdString() ) - , moduleName( QDir( parent->workingPath() ).dirName().toStdString() ) - , configuration( variantMapToPyDict( parent->configuration() ) ) - , m_parent( parent ) -{ } - -void -JobProxy::setprogress( qreal progress ) -{ - if ( progress >= 0.0 && progress <= 1.0 ) - { - m_parent->emitProgress( progress ); - } -} - - -Calamares::GlobalStorage* GlobalStorageProxy::s_gs_instance = nullptr; - -// The special handling for nullptr is only for the testing -// script for the python bindings, which passes in None; -// normal use will have a GlobalStorage from JobQueue::instance() -// passed in. Testing use will leak the allocated GlobalStorage -// object, but that's OK for testing. -GlobalStorageProxy::GlobalStorageProxy( Calamares::GlobalStorage* gs ) - : m_gs( gs ? gs : s_gs_instance ) -{ - if ( !m_gs ) - { - s_gs_instance = new Calamares::GlobalStorage; - m_gs = s_gs_instance; - } -} - -bool -GlobalStorageProxy::contains( const std::string& key ) const -{ - return m_gs->contains( QString::fromStdString( key ) ); -} - -int -GlobalStorageProxy::count() const -{ - return m_gs->count(); -} - -void -GlobalStorageProxy::insert( const std::string& key, const Object& value ) -{ - m_gs->insert( QString::fromStdString( key ), variantFromPyObject( value ) ); -} - -List -GlobalStorageProxy::keys() const -{ - List pyList; - const auto keys = m_gs->keys(); - for ( const QString& key : keys ) - { - pyList.append( key.toStdString() ); - } - return pyList; } - -int -GlobalStorageProxy::remove( const std::string& key ) -{ - const QString gsKey( QString::fromStdString( key ) ); - if ( !m_gs->contains( gsKey ) ) - { - cWarning() << "Unknown GS key" << key.c_str(); - } - return m_gs->remove( gsKey ); -} - -Object -GlobalStorageProxy::value( const std::string& key ) const -{ - const QString gsKey( QString::fromStdString( key ) ); - if ( !m_gs->contains( gsKey ) ) - { - cWarning() << "Unknown GS key" << key.c_str(); - return py::none(); - } - return variantToPyObject( m_gs->value( gsKey ) ); -} - -} // namespace Python -} // namespace Calamares diff --git a/src/libcalamares/python/Api.h b/src/libcalamares/python/Api.h index 9831bca981662ce768fa01410cff707f09efb71d..00f3321907d52682b114768c51a408ffc7704c32 100644 --- a/src/libcalamares/python/Api.h +++ b/src/libcalamares/python/Api.h @@ -1,6 +1,6 @@ /* === This file is part of Calamares - <https://calamares.io> === * - * SPDX-FileCopyrightText: 2023 Adriaan de Groot <groot@kde.org> + * SPDX-FileCopyrightText: 2023, 2024 Adriaan de Groot <groot@kde.org> * SPDX-License-Identifier: GPL-3.0-or-later * * Calamares is Free Software: see the License-Identifier above. @@ -12,21 +12,17 @@ /** @file * - * Contains the API that Python modules use from the Python code - * of that module. This is the C++ side that implements the functions - * imported by the Python code as `import libcalamares`. + * Contains some of the API that Python modules use from the Python code + * of that module. The functions declared here have no complications + * regarding the (Python) types being used. */ -#include "python/Pybind11Helpers.h" +#include "PythonTypes.h" #include <string> namespace Calamares { - -class GlobalStorage; -class PythonJob; - namespace Python __attribute__( ( visibility( "hidden" ) ) ) { std::string obscure( const std::string& string ); @@ -41,65 +37,12 @@ namespace Python __attribute__( ( visibility( "hidden" ) ) ) List gettext_languages(); Object gettext_path(); - int target_env_call( const List& args, const std::string& input, int timeout ); - int check_target_env_call( const List& args, const std::string& input, int timeout ); - std::string check_target_env_output( const List& args, const std::string& input, int timeout ); - - int target_env_process_output( const List& args, const Object& callback, const std::string& input, int timeout ); - int host_env_process_output( const List& args, const Object& callback, const std::string& input, int timeout ); - int mount( const std::string& device_path, - const std::string& mount_point, - const std::string& filesystem_name, - const std::string& options ); - - class Job; - - /** @brief Proxy class in Python for the Calamares Job class - * - * This is available as libcalamares.job in Python code. - */ - class JobProxy - { - public: - explicit JobProxy( Calamares::Python::Job* parent ); - - std::string prettyName; - std::string workingPath; - std::string moduleName; - - Dictionary configuration; - - void setprogress( qreal progress ); - - private: - Calamares::Python::Job* m_parent; - }; - - class GlobalStorageProxy - { - public: - explicit GlobalStorageProxy( Calamares::GlobalStorage* gs ); - - bool contains( const std::string& key ) const; - int count() const; - void insert( const std::string& key, const Object& value ); - List keys() const; - int remove( const std::string& key ); - Object value( const std::string& key ) const; - - // This is a helper for scripts that do not go through - // the JobQueue (i.e. the module testpython script), - // which allocate their own (singleton) GlobalStorage. - static Calamares::GlobalStorage* globalStorageInstance() { return s_gs_instance; } - - private: - Calamares::GlobalStorage* m_gs; - static Calamares::GlobalStorage* s_gs_instance; // See globalStorageInstance() - }; - + const std::string& mount_point, + const std::string& filesystem_name = std::string(), + const std::string& options = std::string() ); -} // namespace Python -} // namespace Calamares +} +} #endif diff --git a/src/libcalamares/python/Logger.h b/src/libcalamares/python/Logger.h deleted file mode 100644 index bf71f9ee945bef56701ca6be821eaa9be666eef2..0000000000000000000000000000000000000000 --- a/src/libcalamares/python/Logger.h +++ /dev/null @@ -1,35 +0,0 @@ -/* === This file is part of Calamares - <https://calamares.io> === - * - * SPDX-FileCopyrightText: 2023 Adriaan de Groot <groot@kde.org> - * SPDX-License-Identifier: GPL-3.0-or-later - * - * Calamares is Free Software: see the License-Identifier above. - * - * - */ - -#ifndef CALAMARES_PYTHON_LOGGER_H -#define CALAMARES_PYTHON_LOGGER_H - -/** @file - * - * Additional logging helpers for pybind11 types. - */ - -#include "utils/Logger.h" - -#include "python/Pybind11Helpers.h" - -#include <string> - -inline QDebug& -operator<<( QDebug& s, const pybind11::handle& h ) -{ -#if QT_VERSION < QT_VERSION_CHECK( 6, 0, 0 ) - return s << QString::fromUtf8( pybind11::str( h ).cast< std::string >().c_str() ); -#else - return s << pybind11::str( h ).cast< std::string >(); -#endif -} - -#endif diff --git a/src/libcalamares/python/Variant.cpp b/src/libcalamares/python/Variant.cpp new file mode 100644 index 0000000000000000000000000000000000000000..22bf51c3cd7afeb686a52522671b4ceed5ef392a --- /dev/null +++ b/src/libcalamares/python/Variant.cpp @@ -0,0 +1,114 @@ +/* === This file is part of Calamares - <https://calamares.io> === + * + * SPDX-FileCopyrightText: 2014 Teo Mrnjavac <teo@kde.org> + * SPDX-FileCopyrightText: 2018-2020, 2024 Adriaan de Groot <groot@kde.org> + * SPDX-License-Identifier: GPL-3.0-or-later + * + * Calamares is Free Software: see the License-Identifier above. + * + */ + +#include "Variant.h" + +#include "PythonTypes.h" +#include "compat/Variant.h" + +namespace +{ + +Calamares::Python::List +variantListToPyList( const QVariantList& variantList ) +{ + Calamares::Python::List pyList; + for ( const QVariant& variant : variantList ) + { + pyList.append( Calamares::Python::variantToPyObject( variant ) ); + } + return pyList; +} + +Calamares::Python::Dictionary +variantHashToPyDict( const QVariantHash& variantHash ) +{ + Calamares::Python::Dictionary pyDict; + for ( auto it = variantHash.constBegin(); it != variantHash.constEnd(); ++it ) + { + pyDict[ Calamares::Python::String( it.key().toStdString() ) ] = Calamares::Python::variantToPyObject( it.value() ); + } + return pyDict; +} + +} + +namespace Calamares +{ +namespace Python +{ + +Dictionary +variantMapToPyDict( const QVariantMap& variantMap ) +{ + Calamares::Python::Dictionary pyDict; + for ( auto it = variantMap.constBegin(); it != variantMap.constEnd(); ++it ) + { + pyDict[ Calamares::Python::String( it.key().toStdString() ) ] = Calamares::Python::variantToPyObject( it.value() ); + } + return pyDict; +} + +Object +variantToPyObject( const QVariant& variant ) +{ + QT_WARNING_PUSH + QT_WARNING_DISABLE_CLANG( "-Wswitch-enum" ) + + // 49 enumeration values not handled + switch ( Calamares::typeOf( variant ) ) + { + case Calamares::MapVariantType: + return variantMapToPyDict( variant.toMap() ); + + case Calamares::HashVariantType: + return variantHashToPyDict( variant.toHash() ); + + case Calamares::ListVariantType: + case Calamares::StringListVariantType: + return variantListToPyList( variant.toList() ); + + case Calamares::IntVariantType: + return Python::Integer( variant.toInt() ); + case Calamares::UIntVariantType: + return Python::Integer( variant.toUInt() ); + + case Calamares::LongLongVariantType: + return Python::Integer( variant.toLongLong() ); + case Calamares::ULongLongVariantType: + return Python::Integer( variant.toULongLong() ); + + case Calamares::DoubleVariantType: + return Python::Float( variant.toDouble() ); + + case Calamares::CharVariantType: +#if QT_VERSION < QT_VERSION_CHECK( 6, 0, 0 ) +#else + // In Qt6, QChar is also available and different from CharVariantType + case QMetaType::Type::QChar: +#endif + case Calamares::StringVariantType: + return Calamares::Python::String( variant.toString().toStdString() ); + + case Calamares::BoolVariantType: + return Python::Boolean( variant.toBool() ); + +#if QT_VERSION < QT_VERSION_CHECK( 6, 0, 0 ) + case QVariant::Invalid: +#endif + default: + return Python::None(); + } + QT_WARNING_POP +} + + + } +} diff --git a/src/libcalamares/python/Variant.h b/src/libcalamares/python/Variant.h new file mode 100644 index 0000000000000000000000000000000000000000..88a5ea65d3c764ea3f0be6777aa3ea4639ff11a7 --- /dev/null +++ b/src/libcalamares/python/Variant.h @@ -0,0 +1,40 @@ +/* === This file is part of Calamares - <https://calamares.io> === + * + * SPDX-FileCopyrightText: 2014 Teo Mrnjavac <teo@kde.org> + * SPDX-FileCopyrightText: 2018-2020, 2024 Adriaan de Groot <groot@kde.org> + * SPDX-License-Identifier: GPL-3.0-or-later + * + * Calamares is Free Software: see the License-Identifier above. + * + */ + +#ifndef CALAMARES_PYTHONVARIANT_H +#define CALAMARES_PYTHONVARIANT_H + +/** + * @file Support for turning QVariant into Python types, and vice-versa + * + * These are helper-functions for converting variants (e.g. values + * in GlobalStorage, or loaded from YAML) into Python. This is not + * public API and is used only inside the Python-job-support code. + */ + +#include "PythonTypes.h" + +#include <QStringList> +#include <QVariant> +#include <QVariantMap> + + +namespace Calamares +{ +namespace Python __attribute__( ( visibility( "hidden" ) ) ) +{ + +Dictionary variantMapToPyDict( const QVariantMap& variantMap ); +Object variantToPyObject( const QVariant& variant ); ///< More generic version of variantMapToPyDict + +} +} + +#endif diff --git a/src/libcalamaresui/modulesystem/PythonJobModule.cpp b/src/libcalamaresui/modulesystem/PythonJobModule.cpp index d9ae7f11556c0be177eff2d134039a66fd4c2e68..95109f14b219f859f260397af8b0249534844886 100644 --- a/src/libcalamaresui/modulesystem/PythonJobModule.cpp +++ b/src/libcalamaresui/modulesystem/PythonJobModule.cpp @@ -12,11 +12,11 @@ #include "CalamaresConfig.h" #ifdef WITH_PYBIND11 -#include "python/PythonJob.h" +#include "pybind11/PythonJob.h" using JobType = Calamares::Python::Job; #elif defined( WITH_BOOST_PYTHON ) // Old Boost::Python version -#include "PythonJob.h" +#include "pyboost/PythonJob.h" using JobType = Calamares::PythonJob; #else #error Python without bindings diff --git a/src/modules/README.md b/src/modules/README.md index 86aa4d726dd94b83ca5d098b2bda39c5ba983c45..e4e56787c06135a832c991115b56cbd28f730531 100644 --- a/src/modules/README.md +++ b/src/modules/README.md @@ -198,6 +198,7 @@ efiSystemPartition|partition |bootloader, fstab|String containing the path extraMounts |mount |unpackfs|List of maps holding metadata for the temporary mountpoints used by the installer fullname |users | |The full username (e.g. "Jane Q. Public") hostname |users | |A string containing the hostname of the new system +luksPassphrase |partition | |Obfuscated passphrase used on luks partition netinstallAdd |packagechooser |netinstall |Data to add to netinstall tree. Same format as netinstall.yaml netinstallSelect |packagechooser |netinstall |List of group names to select in the netinstall tree packageOperations +|packagechooser, netinstall|packages|Operations to perform diff --git a/src/modules/dummycpp/DummyCppJob.cpp b/src/modules/dummycpp/DummyCppJob.cpp index ba8e6ce52341c5b9b13f13130d59b1e7e4bf30dd..0e709d3c3a134636bf61c7bb97c510755df020ca 100644 --- a/src/modules/dummycpp/DummyCppJob.cpp +++ b/src/modules/dummycpp/DummyCppJob.cpp @@ -122,13 +122,25 @@ DummyCppJob::exec() globalStorage->value( "item2" ).toString(), globalStorage->value( "item3" ).toString() ); - emit progress( 0.1 ); + Q_EMIT progress( 0.1 ); cDebug() << "[DUMMYCPP]: " << accumulator; globalStorage->debugDump(); - emit progress( 0.5 ); + Q_EMIT progress( 0.5 ); - QThread::sleep( 3 ); + QThread::sleep( 1 ); + Calamares::System::instance()->targetEnvCall( + QStringList { "ls" }, + QString(), + QString(), + std::chrono::seconds( 1 ) ); // Expect an error because of missing rootMountPoint + + for ( int i = 0; i < 1000000; ++i ) + { + Q_EMIT progress( qreal( i / 1000000.f ) ); + } + + QThread::sleep( 1 ); return Calamares::JobResult::ok(); } diff --git a/src/modules/dummypython/main.py b/src/modules/dummypython/main.py index 65621dd1d1736f93d3cdd2df7f7aaa08c6d3ead3..5ca3455ccb7e63139d14c3000dbd2e8db6375f04 100644 --- a/src/modules/dummypython/main.py +++ b/src/modules/dummypython/main.py @@ -87,6 +87,14 @@ def run(): libcalamares.utils.debug("*** ACTIVITY ***") + # Expect error message that rootMountPoint is not set + libcalamares.utils.target_env_call(["ls"]) + libcalamares.utils.target_env_call("ls") + + # Expect error message can't chroot to /tmp + libcalamares.globalstorage.insert("rootMountPoint", "/tmp") + libcalamares.utils.target_env_call(["ls"]) + sleep(1) million = 1000000 diff --git a/src/modules/partition/gui/ChoicePage.cpp b/src/modules/partition/gui/ChoicePage.cpp index a7665bbbc05da362ec807316862e1550a16bcd76..382c1a7a2604e49ab372cff5d0e88258ed2dac0b 100644 --- a/src/modules/partition/gui/ChoicePage.cpp +++ b/src/modules/partition/gui/ChoicePage.cpp @@ -40,6 +40,7 @@ #include "utils/Gui.h" #include "utils/Logger.h" #include "utils/Retranslator.h" +#include "utils/String.h" #include "utils/Units.h" #include "widgets/PrettyRadioButton.h" @@ -188,7 +189,8 @@ ChoicePage::init( PartitionCoreModule* core ) connect( m_drivesCombo, qOverload< int >( &QComboBox::currentIndexChanged ), this, &ChoicePage::applyDeviceChoice ); connect( m_encryptWidget, &EncryptWidget::stateChanged, this, &ChoicePage::onEncryptWidgetStateChanged ); - connect( m_reuseHomeCheckBox, Calamares::checkBoxStateChangedSignal, this, &ChoicePage::onHomeCheckBoxStateChanged ); + connect( + m_reuseHomeCheckBox, Calamares::checkBoxStateChangedSignal, this, &ChoicePage::onHomeCheckBoxStateChanged ); ChoicePage::applyDeviceChoice(); } @@ -689,6 +691,12 @@ ChoicePage::onHomeCheckBoxStateChanged() void ChoicePage::onLeave() { + Calamares::GlobalStorage* gs = Calamares::JobQueue::instance()->globalStorage(); + const bool useLuksPassphrase = ( m_encryptWidget->state() == EncryptWidget::Encryption::Confirmed ); + const QString storedLuksPassphrase + = useLuksPassphrase ? Calamares::String::obscure( m_encryptWidget->passphrase() ) : QString(); + gs->insert( "luksPassphrase", storedLuksPassphrase ); + if ( m_config->installChoice() == InstallChoice::Alongside ) { if ( m_afterPartitionSplitterWidget->splitPartitionSize() >= 0 diff --git a/src/modules/partition/jobs/ClearMountsJob.cpp b/src/modules/partition/jobs/ClearMountsJob.cpp index 2865d95383cc87ae9fe22be716ceced82c3551d9..2fbafb5dc754d40dccbb18aa512c67d6a3712287 100644 --- a/src/modules/partition/jobs/ClearMountsJob.cpp +++ b/src/modules/partition/jobs/ClearMountsJob.cpp @@ -105,18 +105,22 @@ getSwapsForDevice( const QString& deviceName ) } static inline bool -isControl( const QString& baseName ) -{ - return baseName == "control"; -} - -static inline bool -isFedoraSpecial( const QString& baseName ) +isSpecial( const QString& baseName ) { // Fedora live images use /dev/mapper/live-* internally. We must not // unmount those devices, because they are used by the live image and // because we need /dev/mapper/live-base in the unpackfs module. - return baseName.startsWith( "live-" ); + const bool specialForFedora = baseName.startsWith( "live-" ); + + // Exclude /dev/mapper/control + const bool specialMapperControl = baseName == "control"; + + // When ventoy is used, ventoy uses the /dev/mapper/ventoy device. We + // must not unmount this device, because it is used by the live image + // and because we need /dev/mapper/ventoy in the unpackfs module. + const bool specialVentoy = baseName == "ventoy"; + + return specialForFedora || specialMapperControl || specialVentoy; } /** @brief Returns a list of unneeded crypto devices @@ -135,7 +139,7 @@ getCryptoDevices( const QStringList& mapperExceptions ) for ( const QFileInfo& fi : fiList ) { QString baseName = fi.baseName(); - if ( isControl( baseName ) || isFedoraSpecial( baseName ) || mapperExceptions.contains( baseName ) ) + if ( isSpecial( baseName ) || mapperExceptions.contains( baseName ) ) { continue; } diff --git a/src/modules/partition/jobs/ClearMountsJob.h b/src/modules/partition/jobs/ClearMountsJob.h index fb3aca1e440ff15a59ecf02331c4ba70bd0407e7..44fc81590db402cc0035d25558f9177c0543355f 100644 --- a/src/modules/partition/jobs/ClearMountsJob.h +++ b/src/modules/partition/jobs/ClearMountsJob.h @@ -30,7 +30,8 @@ class Device; * files that should not be closed (e.g. "myvg-mylv"). * * (*) Some exceptions always exist: /dev/mapper/control is never - * closed. /dev/mapper/live-* is never closed. + * closed. /dev/mapper/live-* is never closed. /dev/mapper/ventoy + * is never closed. * */ class ClearMountsJob : public Calamares::Job diff --git a/src/modules/users/CMakeLists.txt b/src/modules/users/CMakeLists.txt index 25e011c8e6f55af581dd47bc6d263e0713666be4..92ea2da647fb2f1d3274ae26abfa78ae375df955 100644 --- a/src/modules/users/CMakeLists.txt +++ b/src/modules/users/CMakeLists.txt @@ -120,9 +120,13 @@ calamares_add_test( calamares_add_test( usershostnametest - SOURCES TestSetHostNameJob.cpp SetHostNameJob.cpp + SOURCES + TestSetHostNameJob.cpp + SetHostNameJob.cpp + ${_users_src} # Build again with test-visibility LIBRARIES ${qtname}::DBus # HostName job can use DBus to systemd + ${USER_EXTRA_LIB} ) calamares_add_test(