Merge branch 'SHADER_HOT_RELOAD' into 'master'

Shaders: Hot reload, togglable by lua debug command

See merge request OpenMW/openmw!2238

(cherry picked from commit 4078f19c74)

8d194a16 Shaders: rudimentary hot reloader on shaders
4e7c1c5b Added break when the operation failed
6b38d622 Added lua bindings to toggle hot reload (disabled by default) or to trigger a reload
31d41252 forgot memory include
f78fa989 fixed include, cleaned comments and indentation
fc8838c7 Renamed lua binding, and use action to avoid concurrency issue
aa51d6de Missing chrono include ?
68d06989 Fixed cyclical included check
b6d7293a Removed weird lines that I thought were necessary to please the compiler
9a475b0c fixed blank lines and missing breaks
cdd95f78 replaced empty function body by default
a1c8dc9d C++17 compat ?
7b78bf4b Fix files with different defines weren't added to the hot reload manager
cc9d4364 includes now work when the same shader has different defines
15751c57 Lua debug api doc
3ab0a991 Hot reload done only once every 200 ms, no point in beeing faster
df69fc76 Post processing shaders now use the same lua commands, no more launcher option...
c71f3508 changed overview.rst of post processing
603b30e1 Added some variable names to make it clearer what their function was
baadc06e Merge branch 'master' into 'SHADER_HOT_RELOAD'
decfbc53 Fix threading issues
b14cc673 adds missing decleration
16a4b571 adds missing include
166717d6 Makes sure threads are only stopped once ,and that they will be re-started
25c1f0ca Renamed variable to fix case issue

apps/launcher/advancedpage.cpp

@@ -152,7 +152,6 @@ bool Launcher::AdvancedPage::loadSettings()
 
         connect(postprocessEnabledCheckBox, SIGNAL(toggled(bool)), this, SLOT(slotPostProcessToggled(bool)));
         loadSettingBool(postprocessEnabledCheckBox, "enabled", "Post Processing");
-        loadSettingBool(postprocessLiveReloadCheckBox, "live reload", "Post Processing");
         loadSettingBool(postprocessTransparentPostpassCheckBox, "transparent postpass", "Post Processing");
         postprocessHDRTimeComboBox->setValue(Settings::Manager::getDouble("auto exposure speed", "Post Processing"));
 
@@ -311,7 +310,6 @@ void Launcher::AdvancedPage::saveSettings()
         saveSettingBool(nightDaySwitchesCheckBox, "day night switches", "Game");
 
         saveSettingBool(postprocessEnabledCheckBox, "enabled", "Post Processing");
-        saveSettingBool(postprocessLiveReloadCheckBox, "live reload", "Post Processing");
         saveSettingBool(postprocessTransparentPostpassCheckBox, "transparent postpass", "Post Processing");
         double hdrExposureTime = postprocessHDRTimeComboBox->value();
         if (hdrExposureTime != Settings::Manager::getDouble("auto exposure speed", "Post Processing"))
@@ -477,7 +475,6 @@ void Launcher::AdvancedPage::slotAnimSourcesToggled(bool checked)
 
 void Launcher::AdvancedPage::slotPostProcessToggled(bool checked)
 {
-    postprocessLiveReloadCheckBox->setEnabled(checked);
     postprocessTransparentPostpassCheckBox->setEnabled(checked);
     postprocessHDRTimeComboBox->setEnabled(checked);
     postprocessHDRTimeLabel->setEnabled(checked);

apps/openmw/mwlua/debugbindings.cpp

@@ -5,6 +5,11 @@
 #include "../mwbase/environment.hpp"
 #include "../mwbase/world.hpp"
 #include "../mwrender/renderingmanager.hpp"
+#include "../mwrender/postprocessor.hpp"
+
+#include <components/resource/resourcesystem.hpp>
+#include <components/resource/scenemanager.hpp>
+#include <components/shader/shadermanager.hpp>
 
 #include <components/lua/luastate.hpp>
 
@@ -46,6 +51,27 @@ namespace MWLua
             });
         };
 
+        api["triggerShaderReload"] = [context]()
+        {
+            context.mLuaManager->addAction([]
+                {
+                    auto world = MWBase::Environment::get().getWorld();
+
+                    world->getRenderingManager()->getResourceSystem()->getSceneManager()->getShaderManager().triggerShaderReload();
+                    world->getPostProcessor()->triggerShaderReload();
+                });
+        };
+
+        api["setShaderHotReloadEnabled"] = [context](bool value)
+        {
+            context.mLuaManager->addAction([value]
+                {
+                    auto world = MWBase::Environment::get().getWorld();
+                    world->getRenderingManager()->getResourceSystem()->getSceneManager()->getShaderManager().setHotReloadEnabled(value);
+                    world->getPostProcessor()->mEnableLiveReload = value;
+                });
+        };
+
         return LuaUtil::makeReadOnly(api);
     }
 }

apps/openmw/mwrender/postprocessor.cpp

@@ -104,6 +104,7 @@ namespace MWRender
 {
     PostProcessor::PostProcessor(RenderingManager& rendering, osgViewer::Viewer* viewer, osg::Group* rootNode, const VFS::Manager* vfs)
         : osg::Group()
+        , mEnableLiveReload(false)
         , mRootNode(rootNode)
         , mSamples(Settings::Manager::getInt("antialiasing", "Video"))
         , mDirty(false)
@@ -111,6 +112,7 @@ namespace MWRender
         , mRendering(rendering)
         , mViewer(viewer)
         , mVFS(vfs)
+        , mTriggerShaderReload(false)
         , mReload(false)
         , mEnabled(false)
         , mUsePostProcessing(false)
@@ -363,10 +365,11 @@ namespace MWRender
 
     void PostProcessor::updateLiveReload()
     {
-        static const bool liveReload = Settings::Manager::getBool("live reload", "Post Processing");
+        if (!mEnableLiveReload && !mTriggerShaderReload)
-        if (!liveReload)
             return;
 
+        mTriggerShaderReload = false;//Done only once
+
         for (auto& technique : mTechniques)
         {
             if (technique->getStatus() == fx::Technique::Status::File_Not_exists)
@@ -859,5 +862,10 @@ namespace MWRender
             return Stereo::Manager::instance().eyeResolution().y();
         return mHeight;
     }
+
+    void PostProcessor::triggerShaderReload()
+    {
+        mTriggerShaderReload = true;
+    }
 }
 

apps/openmw/mwrender/postprocessor.hpp

@@ -180,9 +180,14 @@ namespace MWRender
         int renderWidth() const;
         int renderHeight() const;
 
+        void triggerShaderReload();
+
+        bool mEnableLiveReload;
+
         void loadChain();
         void saveChain();
 
+
     private:
 
         void populateTechniqueFiles();
@@ -226,6 +231,7 @@ namespace MWRender
         osgViewer::Viewer* mViewer;
         const VFS::Manager* mVFS;
 
+        bool mTriggerShaderReload;
         bool mReload;
         bool mEnabled;
         bool mUsePostProcessing;

apps/openmw/mwrender/renderingmanager.cpp

@@ -902,6 +902,8 @@ namespace MWRender
     {
         reportStats();
 
+        mResourceSystem->getSceneManager()->getShaderManager().update(*mViewer);
+
         float rainIntensity = mSky->getPrecipitationAlpha();
         mWater->setRainIntensity(rainIntensity);
 

components/shader/shadermanager.cpp

@@ -5,9 +5,11 @@
 #include <sstream>
 #include <regex>
 #include <filesystem>
-
+#include <set>
+#include <unordered_map>
+#include <chrono>
 #include <osg/Program>
-
+#include <osgViewer/Viewer>
 #include <components/debug/debuglog.hpp>
 #include <components/misc/stringops.hpp>
 #include <components/settings/settings.hpp>
@@ -17,8 +19,11 @@ namespace Shader
 
     ShaderManager::ShaderManager()
     {
+        mHotReloadManager = std::make_unique<HotReloadManager>();
     }
 
+    ShaderManager::~ShaderManager() = default;
+
     void ShaderManager::setShaderPath(const std::string &path)
     {
         mPath = path;
@@ -67,11 +72,12 @@ namespace Shader
 
     // Recursively replaces include statements with the actual source of the included files.
     // Adjusts #line statements accordingly and detects cyclic includes.
-    // includingFiles is the set of files that include this file directly or indirectly, and is intentionally not a reference to allow automatic cleanup.
+    // cycleIncludeChecker is the set of files that include this file directly or indirectly, and is intentionally not a reference to allow automatic cleanup.
-    static bool parseIncludes(const std::filesystem::path& shaderPath, std::string& source, const std::string& fileName, int& fileNumber, std::set<std::filesystem::path> includingFiles)
+    static bool parseIncludes(const std::filesystem::path& shaderPath, std::string& source, const std::string& fileName, int& fileNumber, std::set<std::filesystem::path> cycleIncludeChecker,std::set<std::filesystem::path>& includedFiles)
     {
+        includedFiles.insert(shaderPath / fileName);
         // An include is cyclic if it is being included by itself
-        if (includingFiles.insert(shaderPath/fileName).second == false)
+        if (cycleIncludeChecker.insert(shaderPath/fileName).second == false)
         {
             Log(Debug::Error) << "Shader " << fileName << " error: Detected cyclic #includes";
             return false;
@@ -128,7 +134,7 @@ namespace Shader
             buffer << includeFstream.rdbuf();
             std::string stringRepresentation = buffer.str();
             if (!addLineDirectivesAfterConditionalBlocks(stringRepresentation)
-                || !parseIncludes(shaderPath, stringRepresentation, includeFilename, fileNumber, includingFiles))
+                || !parseIncludes(shaderPath, stringRepresentation, includeFilename, fileNumber, cycleIncludeChecker, includedFiles))
             {
                 Log(Debug::Error) << "In file included from " << fileName << "." << lineNumber;
                 return false;
@@ -355,12 +361,109 @@ namespace Shader
         return true;
     }
 
+    struct HotReloadManager
+    {
+        using KeysHolder = std::set<ShaderManager::MapKey>;
+
+        std::unordered_map<std::string, KeysHolder> mShaderFiles;
+        std::unordered_map<std::string, std::set<std::filesystem::path>> templateIncludedFiles;
+        std::filesystem::file_time_type mLastAutoRecompileTime;
+        bool mHotReloadEnabled;
+        bool mTriggerReload;
+
+        HotReloadManager()
+        {
+            mTriggerReload = false;
+            mHotReloadEnabled = false;
+            mLastAutoRecompileTime = std::filesystem::file_time_type::clock::now();
+        }
+
+        void addShaderFiles(const std::string& templateName,const ShaderManager::DefineMap& defines )
+        {
+            const std::set<std::filesystem::path>& shaderFiles = templateIncludedFiles[templateName];
+            for (const std::filesystem::path& file : shaderFiles)
+            {
+                mShaderFiles[file.string()].insert(std::make_pair(templateName, defines));
+            }
+        }
+
+        void update(ShaderManager& Manager,osgViewer::Viewer& viewer)
+        {
+            auto timeSinceLastCheckMillis = std::chrono::duration_cast<std::chrono::milliseconds>(std::filesystem::file_time_type::clock::now() - mLastAutoRecompileTime);
+            if ((mHotReloadEnabled && timeSinceLastCheckMillis.count() > 200) || mTriggerReload == true)
+            {
+                reloadTouchedShaders(Manager, viewer);
+            }
+            mTriggerReload = false;
+        }
+
+        void reloadTouchedShaders(ShaderManager& Manager, osgViewer::Viewer& viewer)
+        {
+            bool threadsRunningToStop = false;
+            for (auto& [pathShaderToTest,  shaderKeys]: mShaderFiles)
+            {
+
+                std::filesystem::file_time_type write_time = std::filesystem::last_write_time(pathShaderToTest);
+                if (write_time.time_since_epoch() > mLastAutoRecompileTime.time_since_epoch())
+                {
+                    if (!threadsRunningToStop)
+                    {
+                        threadsRunningToStop = viewer.areThreadsRunning();
+                        if (threadsRunningToStop)
+                            viewer.stopThreading();
+                    }
+
+                    for (const auto& [templateName, shaderDefines]: shaderKeys)
+                    {
+                        ShaderManager::ShaderMap::iterator shaderIt = Manager.mShaders.find(std::make_pair(templateName, shaderDefines));
+
+                        ShaderManager::TemplateMap::iterator templateIt = Manager.mShaderTemplates.find(templateName); //Can't be Null, if we're here it means the template was added
+                        std::string& shaderSource = templateIt->second;
+                        std::set<std::filesystem::path> insertedPaths;
+                        std::filesystem::path path = (std::filesystem::path(Manager.mPath) / templateName);
+                        std::ifstream stream;
+                        stream.open(path);
+                        if (stream.fail())
+                        {
+                            Log(Debug::Error) << "Failed to open " << path.string();
+                        }
+                        std::stringstream buffer;
+                        buffer << stream.rdbuf();
+
+                        // parse includes
+                        int fileNumber = 1;
+                        std::string source = buffer.str();
+                        if (!addLineDirectivesAfterConditionalBlocks(source)
+                            || !parseIncludes(std::filesystem::path(Manager.mPath), source, templateName, fileNumber, {}, insertedPaths))
+                        {
+                            break;
+                        }
+                        shaderSource = source;
+
+                        std::vector<std::string> linkedShaderNames;
+                        if (!Manager.createSourceFromTemplate(shaderSource, linkedShaderNames, templateName, shaderDefines))
+                        {
+                            break;
+                        }
+                        shaderIt->second->setShaderSource(shaderSource);
+                            
+                    }
+                }
+            }
+            if (threadsRunningToStop)
+                viewer.startThreading();
+            mLastAutoRecompileTime = std::filesystem::file_time_type::clock::now();
+        }
+    };
+
     osg::ref_ptr<osg::Shader> ShaderManager::getShader(const std::string &templateName, const ShaderManager::DefineMap &defines, osg::Shader::Type shaderType)
     {
         std::unique_lock<std::mutex> lock(mMutex);
 
         // read the template if we haven't already
         TemplateMap::iterator templateIt = mShaderTemplates.find(templateName);
+        std::set<std::filesystem::path> insertedPaths;
+
         if (templateIt == mShaderTemplates.end())
         {
             std::filesystem::path path = (std::filesystem::path(mPath) / templateName);
@@ -378,9 +481,9 @@ namespace Shader
             int fileNumber = 1;
             std::string source = buffer.str();
             if (!addLineDirectivesAfterConditionalBlocks(source)
-                || !parseIncludes(std::filesystem::path(mPath), source, templateName, fileNumber, {}))
+                || !parseIncludes(std::filesystem::path(mPath), source, templateName, fileNumber, {}, insertedPaths))
                 return nullptr;
-
+            mHotReloadManager->templateIncludedFiles[templateName] = insertedPaths;
             templateIt = mShaderTemplates.insert(std::make_pair(templateName, source)).first;
         }
 
@@ -403,6 +506,8 @@ namespace Shader
             static unsigned int counter = 0;
             shader->setName(Misc::StringUtils::format("%u %s", counter++, templateName));
 
+            mHotReloadManager->addShaderFiles(templateName, defines);
+
             lock.unlock();
             getLinkedShaders(shader, linkedShaderNames, defines);
             lock.lock();
@@ -535,4 +640,19 @@ namespace Shader
         return unit;
     }
 
+    void ShaderManager::update(osgViewer::Viewer& viewer)
+    {
+        mHotReloadManager->update(*this, viewer);
+    }
+
+    void ShaderManager::setHotReloadEnabled(bool value)
+    {
+        mHotReloadManager->mHotReloadEnabled = value;
+    }
+
+    void ShaderManager::triggerShaderReload()
+    {
+        mHotReloadManager->mTriggerReload = true;
+    }
+
 }

components/shader/shadermanager.hpp

@@ -6,22 +6,28 @@
 #include <mutex>
 #include <vector>
 #include <array>
+#include <memory>
 
 #include <osg/ref_ptr>
 
 #include <osg/Shader>
 #include <osg/Program>
 
+namespace osgViewer {
+    class Viewer;
+}
+
 namespace Shader
 {
-
+    struct HotReloadManager;
     /// @brief Reads shader template files and turns them into a concrete shader, based on a list of define's.
     /// @par Shader templates can get the value of a define with the syntax @define.
     class ShaderManager
     {
     public:
-
+        friend HotReloadManager;
         ShaderManager();
+        ~ShaderManager();
 
         void setShaderPath(const std::string& path);
 
@@ -67,6 +73,9 @@ namespace Shader
 
         int reserveGlobalTextureUnits(Slot slot);
 
+        void update(osgViewer::Viewer& viewer);
+        void setHotReloadEnabled(bool value);
+        void triggerShaderReload();
     private:
         void getLinkedShaders(osg::ref_ptr<osg::Shader> shader, const std::vector<std::string>& linkedShaderNames, const DefineMap& defines);
         void addLinkedShaders(osg::ref_ptr<osg::Shader> shader, osg::ref_ptr<osg::Program> program);
@@ -96,7 +105,7 @@ namespace Shader
 
         int mMaxTextureUnits = 0;
         int mReservedTextureUnits = 0;
-
+        std::unique_ptr<HotReloadManager> mHotReloadManager;
         std::array<int, 2> mReservedTextureUnitsBySlot = {-1, -1};
     };
 

docs/source/reference/postprocessing/overview.rst

@@ -45,8 +45,9 @@ Hot Reloading
 =============
 
 It is possible to modify a shader without restarting OpenMW, :ref:`live reload`
-must be enabled in ``settings.cfg``. Whenever a file is modified and saved, the
+must be enabled by using the lua command `debug.setShaderHotReloadEnabled(true)`. 
-shader will automatically reload in game. This allows shaders to be written in a
+Whenever a file is modified and saved, the shader will automatically reload in game.
-text editor you are comfortable with. The only restriction is that the VFS is not
+You can also trigger a single reload using `debug.triggerShaderReload()`
-aware of new files or changes in non-shader files, so new shaders and localization
+This allows shaders to be written in a text editor you are comfortable with. 
-strings can not be used.
+The only restriction is that the VFS is not aware of new files or changes in non-shader files, 
+so new shaders and localization strings can not be used.

files/lua_api/openmw/debug.lua

@@ -41,4 +41,12 @@
 -- @function [parent=#debug] setNavMeshRenderMode
 -- @param #NAV_MESH_RENDER_MODE value
 
+---
+-- Enable/disable automatic reload of modified shaders
+-- @function [parent=#debug] setShaderHotReloadEnabled
+-- @param #bool value
+
+---
+-- To reload modified shaders
+-- @function [parent=#debug] triggerShaderReload
 return nil

files/ui/advancedpage.ui

@@ -670,19 +670,6 @@
                   <property name="leftMargin">
                     <number>20</number>
                   </property>
-                  <item>
-                    <widget class="QCheckBox" name="postprocessLiveReloadCheckBox">
-                      <property name="enabled">
-                        <bool>false</bool>
-                      </property>
-                      <property name="toolTip">
-                        <string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;Debug Mode. Automatically reload active shaders when they are modified on filesystem.&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
-                      </property>
-                      <property name="text">
-                        <string>Live reload</string>
-                      </property>
-                    </widget>
-                  </item>
                   <item>
                     <widget class="QCheckBox" name="postprocessTransparentPostpassCheckBox">
                       <property name="enabled">