Merge branch 'SHADER_HOT_RELOAD' into 'master'

Shaders: Hot reload, togglable by lua debug command

See merge request OpenMW/openmw!2238

apps/launcher/advancedpage.cpp

@@ -152,7 +152,6 @@ bool Launcher::AdvancedPage::loadSettings()
 
         connect(postprocessEnabledCheckBox, SIGNAL(toggled(bool)), this, SLOT(slotPostProcessToggled(bool)));
         loadSettingBool(postprocessEnabledCheckBox, "enabled", "Post Processing");
-        loadSettingBool(postprocessLiveReloadCheckBox, "live reload", "Post Processing");
         loadSettingBool(postprocessTransparentPostpassCheckBox, "transparent postpass", "Post Processing");
         postprocessHDRTimeComboBox->setValue(Settings::Manager::getDouble("auto exposure speed", "Post Processing"));
 
@@ -311,7 +310,6 @@ void Launcher::AdvancedPage::saveSettings()
         saveSettingBool(nightDaySwitchesCheckBox, "day night switches", "Game");
 
         saveSettingBool(postprocessEnabledCheckBox, "enabled", "Post Processing");
-        saveSettingBool(postprocessLiveReloadCheckBox, "live reload", "Post Processing");
         saveSettingBool(postprocessTransparentPostpassCheckBox, "transparent postpass", "Post Processing");
         double hdrExposureTime = postprocessHDRTimeComboBox->value();
         if (hdrExposureTime != Settings::Manager::getDouble("auto exposure speed", "Post Processing"))
@@ -477,7 +475,6 @@ void Launcher::AdvancedPage::slotAnimSourcesToggled(bool checked)
 
 void Launcher::AdvancedPage::slotPostProcessToggled(bool checked)
 {
-    postprocessLiveReloadCheckBox->setEnabled(checked);
     postprocessTransparentPostpassCheckBox->setEnabled(checked);
     postprocessHDRTimeComboBox->setEnabled(checked);
     postprocessHDRTimeLabel->setEnabled(checked);

apps/openmw/mwlua/debugbindings.cpp

@@ -5,6 +5,11 @@
 #include "../mwbase/environment.hpp"
 #include "../mwbase/world.hpp"
 #include "../mwrender/renderingmanager.hpp"
+#include "../mwrender/postprocessor.hpp"
+
+#include <components/resource/resourcesystem.hpp>
+#include <components/resource/scenemanager.hpp>
+#include <components/shader/shadermanager.hpp>
 
 #include <components/lua/luastate.hpp>
 
@@ -46,6 +51,27 @@ namespace MWLua
             });
         };
 
+        api["triggerShaderReload"] = [context]()
+        {
+            context.mLuaManager->addAction([]
+                {
+                    auto world = MWBase::Environment::get().getWorld();
+
+                    world->getRenderingManager()->getResourceSystem()->getSceneManager()->getShaderManager().triggerShaderReload();
+                    world->getPostProcessor()->triggerShaderReload();
+                });
+        };
+
+        api["setShaderHotReloadEnabled"] = [context](bool value)
+        {
+            context.mLuaManager->addAction([value]
+                {
+                    auto world = MWBase::Environment::get().getWorld();
+                    world->getRenderingManager()->getResourceSystem()->getSceneManager()->getShaderManager().setHotReloadEnabled(value);
+                    world->getPostProcessor()->mEnableLiveReload = value;
+                });
+        };
+
         return LuaUtil::makeReadOnly(api);
     }
 }

apps/openmw/mwrender/postprocessor.cpp

@@ -105,6 +105,7 @@ namespace MWRender
 {
     PostProcessor::PostProcessor(RenderingManager& rendering, osgViewer::Viewer* viewer, osg::Group* rootNode, const VFS::Manager* vfs)
         : osg::Group()
+        , mEnableLiveReload(false)
         , mRootNode(rootNode)
         , mSamples(Settings::Manager::getInt("antialiasing", "Video"))
         , mDirty(false)
@@ -112,6 +113,7 @@ namespace MWRender
         , mRendering(rendering)
         , mViewer(viewer)
         , mVFS(vfs)
+        , mTriggerShaderReload(false)
         , mReload(false)
         , mEnabled(false)
         , mUsePostProcessing(false)
@@ -364,10 +366,11 @@ namespace MWRender
 
     void PostProcessor::updateLiveReload()
     {
-        static const bool liveReload = Settings::Manager::getBool("live reload", "Post Processing");
+        if (!mEnableLiveReload && !mTriggerShaderReload)
-        if (!liveReload)
             return;
 
+        mTriggerShaderReload = false;//Done only once
+
         for (auto& technique : mTechniques)
         {
             if (technique->getStatus() == fx::Technique::Status::File_Not_exists)
@@ -860,5 +863,10 @@ namespace MWRender
             return Stereo::Manager::instance().eyeResolution().y();
         return mHeight;
     }
+
+    void PostProcessor::triggerShaderReload()
+    {
+        mTriggerShaderReload = true;
+    }
 }
 

apps/openmw/mwrender/postprocessor.hpp

@@ -180,9 +180,14 @@ namespace MWRender
         int renderWidth() const;
         int renderHeight() const;
 
+        void triggerShaderReload();
+
+        bool mEnableLiveReload;
+
         void loadChain();
         void saveChain();
 
+
     private:
 
         void populateTechniqueFiles();
@@ -226,6 +231,7 @@ namespace MWRender
         osgViewer::Viewer* mViewer;
         const VFS::Manager* mVFS;
 
+        bool mTriggerShaderReload;
         bool mReload;
         bool mEnabled;
         bool mUsePostProcessing;

apps/openmw/mwrender/renderingmanager.cpp

@@ -903,6 +903,8 @@ namespace MWRender
     {
         reportStats();
 
+        mResourceSystem->getSceneManager()->getShaderManager().update(*mViewer);
+
         float rainIntensity = mSky->getPrecipitationAlpha();
         mWater->setRainIntensity(rainIntensity);
 

components/shader/shadermanager.cpp

@@ -5,9 +5,11 @@
 #include <sstream>
 #include <regex>
 #include <filesystem>
-
+#include <set>
+#include <unordered_map>
+#include <chrono>
 #include <osg/Program>
-
+#include <osgViewer/Viewer>
 #include <components/debug/debuglog.hpp>
 #include <components/misc/strings/algorithm.hpp>
 #include <components/misc/strings/format.hpp>
@@ -18,8 +20,11 @@ namespace Shader
 
     ShaderManager::ShaderManager()
     {
+        mHotReloadManager = std::make_unique<HotReloadManager>();
     }
 
+    ShaderManager::~ShaderManager() = default;
+
     void ShaderManager::setShaderPath(const std::string &path)
     {
         mPath = path;
@@ -68,11 +73,12 @@ namespace Shader
 
     // Recursively replaces include statements with the actual source of the included files.
     // Adjusts #line statements accordingly and detects cyclic includes.
-    // includingFiles is the set of files that include this file directly or indirectly, and is intentionally not a reference to allow automatic cleanup.
+    // cycleIncludeChecker is the set of files that include this file directly or indirectly, and is intentionally not a reference to allow automatic cleanup.
-    static bool parseIncludes(const std::filesystem::path& shaderPath, std::string& source, const std::string& fileName, int& fileNumber, std::set<std::filesystem::path> includingFiles)
+    static bool parseIncludes(const std::filesystem::path& shaderPath, std::string& source, const std::string& fileName, int& fileNumber, std::set<std::filesystem::path> cycleIncludeChecker,std::set<std::filesystem::path>& includedFiles)
     {
+        includedFiles.insert(shaderPath / fileName);
         // An include is cyclic if it is being included by itself
-        if (includingFiles.insert(shaderPath/fileName).second == false)
+        if (cycleIncludeChecker.insert(shaderPath/fileName).second == false)
         {
             Log(Debug::Error) << "Shader " << fileName << " error: Detected cyclic #includes";
             return false;
@@ -129,7 +135,7 @@ namespace Shader
             buffer << includeFstream.rdbuf();
             std::string stringRepresentation = buffer.str();
             if (!addLineDirectivesAfterConditionalBlocks(stringRepresentation)
-                || !parseIncludes(shaderPath, stringRepresentation, includeFilename, fileNumber, includingFiles))
+                || !parseIncludes(shaderPath, stringRepresentation, includeFilename, fileNumber, cycleIncludeChecker, includedFiles))
             {
                 Log(Debug::Error) << "In file included from " << fileName << "." << lineNumber;
                 return false;
@@ -356,12 +362,109 @@ namespace Shader
         return true;
     }
 
+    struct HotReloadManager
+    {
+        using KeysHolder = std::set<ShaderManager::MapKey>;
+
+        std::unordered_map<std::string, KeysHolder> mShaderFiles;
+        std::unordered_map<std::string, std::set<std::filesystem::path>> templateIncludedFiles;
+        std::filesystem::file_time_type mLastAutoRecompileTime;
+        bool mHotReloadEnabled;
+        bool mTriggerReload;
+
+        HotReloadManager()
+        {
+            mTriggerReload = false;
+            mHotReloadEnabled = false;
+            mLastAutoRecompileTime = std::filesystem::file_time_type::clock::now();
+        }
+
+        void addShaderFiles(const std::string& templateName,const ShaderManager::DefineMap& defines )
+        {
+            const std::set<std::filesystem::path>& shaderFiles = templateIncludedFiles[templateName];
+            for (const std::filesystem::path& file : shaderFiles)
+            {
+                mShaderFiles[file.string()].insert(std::make_pair(templateName, defines));
+            }
+        }
+
+        void update(ShaderManager& Manager,osgViewer::Viewer& viewer)
+        {
+            auto timeSinceLastCheckMillis = std::chrono::duration_cast<std::chrono::milliseconds>(std::filesystem::file_time_type::clock::now() - mLastAutoRecompileTime);
+            if ((mHotReloadEnabled && timeSinceLastCheckMillis.count() > 200) || mTriggerReload == true)
+            {
+                reloadTouchedShaders(Manager, viewer);
+            }
+            mTriggerReload = false;
+        }
+
+        void reloadTouchedShaders(ShaderManager& Manager, osgViewer::Viewer& viewer)
+        {
+            bool threadsRunningToStop = false;
+            for (auto& [pathShaderToTest,  shaderKeys]: mShaderFiles)
+            {
+
+                std::filesystem::file_time_type write_time = std::filesystem::last_write_time(pathShaderToTest);
+                if (write_time.time_since_epoch() > mLastAutoRecompileTime.time_since_epoch())
+                {
+                    if (!threadsRunningToStop)
+                    {
+                        threadsRunningToStop = viewer.areThreadsRunning();
+                        if (threadsRunningToStop)
+                            viewer.stopThreading();
+                    }
+
+                    for (const auto& [templateName, shaderDefines]: shaderKeys)
+                    {
+                        ShaderManager::ShaderMap::iterator shaderIt = Manager.mShaders.find(std::make_pair(templateName, shaderDefines));
+
+                        ShaderManager::TemplateMap::iterator templateIt = Manager.mShaderTemplates.find(templateName); //Can't be Null, if we're here it means the template was added
+                        std::string& shaderSource = templateIt->second;
+                        std::set<std::filesystem::path> insertedPaths;
+                        std::filesystem::path path = (std::filesystem::path(Manager.mPath) / templateName);
+                        std::ifstream stream;
+                        stream.open(path);
+                        if (stream.fail())
+                        {
+                            Log(Debug::Error) << "Failed to open " << path.string();
+                        }
+                        std::stringstream buffer;
+                        buffer << stream.rdbuf();
+
+                        // parse includes
+                        int fileNumber = 1;
+                        std::string source = buffer.str();
+                        if (!addLineDirectivesAfterConditionalBlocks(source)
+                            || !parseIncludes(std::filesystem::path(Manager.mPath), source, templateName, fileNumber, {}, insertedPaths))
+                        {
+                            break;
+                        }
+                        shaderSource = source;
+
+                        std::vector<std::string> linkedShaderNames;
+                        if (!Manager.createSourceFromTemplate(shaderSource, linkedShaderNames, templateName, shaderDefines))
+                        {
+                            break;
+                        }
+                        shaderIt->second->setShaderSource(shaderSource);
+                            
+                    }
+                }
+            }
+            if (threadsRunningToStop)
+                viewer.startThreading();
+            mLastAutoRecompileTime = std::filesystem::file_time_type::clock::now();
+        }
+    };
+
     osg::ref_ptr<osg::Shader> ShaderManager::getShader(const std::string &templateName, const ShaderManager::DefineMap &defines, osg::Shader::Type shaderType)
     {
         std::unique_lock<std::mutex> lock(mMutex);
 
         // read the template if we haven't already
         TemplateMap::iterator templateIt = mShaderTemplates.find(templateName);
+        std::set<std::filesystem::path> insertedPaths;
+
         if (templateIt == mShaderTemplates.end())
         {
             std::filesystem::path path = (std::filesystem::path(mPath) / templateName);
@@ -379,9 +482,9 @@ namespace Shader
             int fileNumber = 1;
             std::string source = buffer.str();
             if (!addLineDirectivesAfterConditionalBlocks(source)
-                || !parseIncludes(std::filesystem::path(mPath), source, templateName, fileNumber, {}))
+                || !parseIncludes(std::filesystem::path(mPath), source, templateName, fileNumber, {}, insertedPaths))
                 return nullptr;
-
+            mHotReloadManager->templateIncludedFiles[templateName] = insertedPaths;
             templateIt = mShaderTemplates.insert(std::make_pair(templateName, source)).first;
         }
 
@@ -404,6 +507,8 @@ namespace Shader
             static unsigned int counter = 0;
             shader->setName(Misc::StringUtils::format("%u %s", counter++, templateName));
 
+            mHotReloadManager->addShaderFiles(templateName, defines);
+
             lock.unlock();
             getLinkedShaders(shader, linkedShaderNames, defines);
             lock.lock();
@@ -536,4 +641,19 @@ namespace Shader
         return unit;
     }
 
+    void ShaderManager::update(osgViewer::Viewer& viewer)
+    {
+        mHotReloadManager->update(*this, viewer);
+    }
+
+    void ShaderManager::setHotReloadEnabled(bool value)
+    {
+        mHotReloadManager->mHotReloadEnabled = value;
+    }
+
+    void ShaderManager::triggerShaderReload()
+    {
+        mHotReloadManager->mTriggerReload = true;
+    }
+
 }

components/shader/shadermanager.hpp

@@ -6,22 +6,28 @@
 #include <mutex>
 #include <vector>
 #include <array>
+#include <memory>
 
 #include <osg/ref_ptr>
 
 #include <osg/Shader>
 #include <osg/Program>
 
+namespace osgViewer {
+    class Viewer;
+}
+
 namespace Shader
 {
-
+    struct HotReloadManager;
     /// @brief Reads shader template files and turns them into a concrete shader, based on a list of define's.
     /// @par Shader templates can get the value of a define with the syntax @define.
     class ShaderManager
     {
     public:
-
+        friend HotReloadManager;
         ShaderManager();
+        ~ShaderManager();
 
         void setShaderPath(const std::string& path);
 
@@ -67,6 +73,9 @@ namespace Shader
 
         int reserveGlobalTextureUnits(Slot slot);
 
+        void update(osgViewer::Viewer& viewer);
+        void setHotReloadEnabled(bool value);
+        void triggerShaderReload();
     private:
         void getLinkedShaders(osg::ref_ptr<osg::Shader> shader, const std::vector<std::string>& linkedShaderNames, const DefineMap& defines);
         void addLinkedShaders(osg::ref_ptr<osg::Shader> shader, osg::ref_ptr<osg::Program> program);
@@ -96,7 +105,7 @@ namespace Shader
 
         int mMaxTextureUnits = 0;
         int mReservedTextureUnits = 0;
-
+        std::unique_ptr<HotReloadManager> mHotReloadManager;
         std::array<int, 2> mReservedTextureUnitsBySlot = {-1, -1};
     };
 

docs/source/reference/postprocessing/overview.rst

@@ -45,8 +45,9 @@ Hot Reloading
 =============
 
 It is possible to modify a shader without restarting OpenMW, :ref:`live reload`
-must be enabled in ``settings.cfg``. Whenever a file is modified and saved, the
+must be enabled by using the lua command `debug.setShaderHotReloadEnabled(true)`. 
-shader will automatically reload in game. This allows shaders to be written in a
+Whenever a file is modified and saved, the shader will automatically reload in game.
-text editor you are comfortable with. The only restriction is that the VFS is not
+You can also trigger a single reload using `debug.triggerShaderReload()`
-aware of new files or changes in non-shader files, so new shaders and localization
+This allows shaders to be written in a text editor you are comfortable with. 
-strings can not be used.
+The only restriction is that the VFS is not aware of new files or changes in non-shader files, 
+so new shaders and localization strings can not be used.

files/lua_api/openmw/debug.lua

@@ -41,4 +41,12 @@
 -- @function [parent=#debug] setNavMeshRenderMode
 -- @param #NAV_MESH_RENDER_MODE value
 
+---
+-- Enable/disable automatic reload of modified shaders
+-- @function [parent=#debug] setShaderHotReloadEnabled
+-- @param #bool value
+
+---
+-- To reload modified shaders
+-- @function [parent=#debug] triggerShaderReload
 return nil

files/ui/advancedpage.ui

@@ -670,19 +670,6 @@
                   <property name="leftMargin">
                     <number>20</number>
                   </property>
-                  <item>
-                    <widget class="QCheckBox" name="postprocessLiveReloadCheckBox">
-                      <property name="enabled">
-                        <bool>false</bool>
-                      </property>
-                      <property name="toolTip">
-                        <string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;Debug Mode. Automatically reload active shaders when they are modified on filesystem.&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
-                      </property>
-                      <property name="text">
-                        <string>Live reload</string>
-                      </property>
-                    </widget>
-                  </item>
                   <item>
                     <widget class="QCheckBox" name="postprocessTransparentPostpassCheckBox">
                       <property name="enabled">