asset_storage.h

Go to the documentation of this file.

#pragma once
 
#include "asset_handle.h"
#include "asset_flags.h"
 
#include <context/context.hpp>
#include <hpp/event.hpp>
 
#include <cassert>
#include <functional>
#include <map>
#include <mutex>
#include <unordered_map>
 
#include <reflection/registration.h>
 
namespace unravel
{
 
struct asset_importer_meta : crtp_meta_type<asset_importer_meta>
{
    virtual ~asset_importer_meta() = default;
};
 
struct texture_importer_meta : crtp_meta_type<texture_importer_meta, asset_importer_meta>
{
    enum class texture_type
    {
        automatic,
        normal_map,
        equirect,
    };
 
    enum class compression_quality
    {
        project_default,
        none,
        low_quality,
        normal_quality,
        high_quality
    };
 
    enum class texture_size
    {
        project_default,
        size_32,
        size_64,
        size_128,
        size_256,
        size_512,
        size_1024,
        size_2048,
        size_4096,
        size_8192,
        size_16384
        
    };
 
    texture_type type{texture_type::automatic};
    bool generate_mipmaps{true};
    bool invert_normal_y{false};
 
    struct quality_meta
    {
        texture_size max_size{texture_size::project_default};
        compression_quality compression{compression_quality::project_default};
 
    } quality;
 
 
};
 
struct mesh_importer_meta : crtp_meta_type<mesh_importer_meta, asset_importer_meta>
{
    struct model_meta
    {
        bool import_meshes{true};
        bool weld_vertices{true};
        bool optimize_meshes{true};
        bool split_large_meshes{true};
        bool find_degenerates{true};
        bool find_invalid_data{true};
        
        bool generate_lods{true};
        float lod_target_error{0.01f};
    } model;
 
    struct rig_meta
    {
 
    } rig;
 
    struct animations_meta
    {
        bool import_animations{true};
 
    } animations;
 
    struct materials_meta
    {
        bool import_materials{true};
        bool remove_redundant_materials{true};
 
    } materials;
};
 
struct animation_importer_meta : crtp_meta_type<animation_importer_meta, asset_importer_meta>
{
 
    struct root_motion_meta
    {
        bool keep_position_y{true};
        bool keep_position_xz{};
        bool keep_rotation{};
 
        bool keep_in_place{};
 
    } root_motion;
};
 
struct audio_importer_meta : crtp_meta_type<audio_importer_meta, asset_importer_meta>
{
    bool force_to_mono{false};
};
 
//-----------------------------------------------------------------------------
// Asset header info -- lightweight per-type metadata readable without full load.
// Populated during compilation, stored in .meta files (editor only).
//-----------------------------------------------------------------------------
 
struct asset_header_info : crtp_meta_type<asset_header_info>
{
    virtual ~asset_header_info() = default;
    size_t file_size{};
};
 
struct texture_header_info : crtp_meta_type<texture_header_info, asset_header_info>
{
    uint32_t width{};
    uint32_t height{};
    uint16_t depth{1};
    uint16_t num_layers{1};
    uint16_t num_mips{1};
    std::string format{};
};
 
struct mesh_header_info : crtp_meta_type<mesh_header_info, asset_header_info>
{
    uint32_t vertex_count{};
    uint32_t index_count{};
    uint32_t submesh_count{};
    uint32_t lod_count{};
};
 
struct animation_header_info : crtp_meta_type<animation_header_info, asset_header_info>
{
    float duration{};
    uint32_t channel_count{};
    float sample_rate{};
};
 
struct audio_header_info : crtp_meta_type<audio_header_info, asset_header_info>
{
    float duration{};
    uint32_t sample_rate{};
    uint16_t channels{};
};
 
struct asset_meta
{
    hpp::uuid uid{};
    std::string type{};
    std::shared_ptr<asset_importer_meta> importer;
    std::shared_ptr<asset_header_info> header;
};
 
class asset_database
{
public:
    struct meta
    {
        std::string location{};
        asset_meta meta;
    };
 
    using database_t = std::map<hpp::uuid, meta>;
 
    auto get_database() const -> const database_t&
    {
        return asset_meta_;
    }
 
    void set_database(const database_t& rhs)
    {
        std::lock_guard<std::mutex> lock(asset_mutex_);
        asset_meta_ = rhs;
    }
 
    void remove_all()
    {
        std::lock_guard<std::mutex> lock(asset_mutex_);
        asset_meta_.clear();
    }
 
    auto add_asset(const std::string& location, const asset_meta& meta, bool override) -> hpp::uuid
    {
        std::lock_guard<std::mutex> lock(asset_mutex_);
 
        auto existing = find_entry_by_location_unlocked(location);
        if(existing != asset_meta_.end())
        {
            if(!override)
            {
                return existing->first;
            }
 
            // Update in place, preserving the existing UUID so that other
            // assets that reference it (e.g. materials → textures) don't get
            // orphaned by an override pass.
            existing->second.location = location;
            existing->second.meta = meta;
            existing->second.meta.uid = existing->first;
            return existing->first;
        }
 
        auto& metainfo = asset_meta_[meta.uid];
        metainfo.location = location;
        metainfo.meta = meta;
        // APPLOG_TRACE("{} - {} -> {}", __func__, hpp::to_string(metainfo.meta.uid), location);
        return metainfo.meta.uid;
    }
 
    auto get_uuid(const std::string& location) const -> const hpp::uuid&
    {
        std::lock_guard<std::mutex> lock(asset_mutex_);
 
        auto it = find_entry_by_location_unlocked(location);
        if(it != asset_meta_.end())
        {
            return it->first;
        }
 
        static const hpp::uuid uid;
        return uid;
    }
 
    auto get_metadata(const std::string& location) const -> const meta&
    {
        std::lock_guard<std::mutex> lock(asset_mutex_);
 
        auto it = find_entry_by_location_unlocked(location);
        if(it != asset_meta_.end())
        {
            return it->second;
        }
 
        static const meta empty;
        return empty;
    }
 
    auto get_metadata(const hpp::uuid& id) const -> const meta&
    {
        std::lock_guard<std::mutex> lock(asset_mutex_);
 
        auto it = asset_meta_.find(id);
        if(it == asset_meta_.end())
        {
            static const meta empty;
            return empty;
        }
 
        return it->second;
    }
 
    void rename_asset(const std::string& key, const std::string& new_key)
    {
        std::lock_guard<std::mutex> lock(asset_mutex_);
        for(auto& kvp : asset_meta_)
        {
            auto& uid = kvp.first;
            auto& metainfo = kvp.second;
            if(metainfo.location == key)
            {
                APPLOG_TRACE("{}::{} - {} -> {}", __func__, hpp::to_string(uid), key, new_key);
 
                metainfo.location = new_key;
            }
        }
    }
 
    void remove_asset(const std::string& key)
    {
        std::lock_guard<std::mutex> lock(asset_mutex_);
        for(auto& kvp : asset_meta_)
        {
            auto& uid = kvp.first;
            auto& metainfo = kvp.second;
            if(metainfo.location == key)
            {
                APPLOG_TRACE("{}::{} - {}", __func__, hpp::to_string(uid), key);
 
                asset_meta_.erase(uid);
                return;
            }
        }
    }
 
private:
    auto find_entry_by_location_unlocked(const std::string& location) -> database_t::iterator
    {
        for(auto it = asset_meta_.begin(); it != asset_meta_.end(); ++it)
        {
            if(it->second.location == location)
            {
                return it;
            }
        }
        return asset_meta_.end();
    }
 
    auto find_entry_by_location_unlocked(const std::string& location) const -> database_t::const_iterator
    {
        for(auto it = asset_meta_.cbegin(); it != asset_meta_.cend(); ++it)
        {
            if(it->second.location == location)
            {
                return it;
            }
        }
        return asset_meta_.cend();
    }
 
    mutable std::mutex asset_mutex_{};
    database_t asset_meta_{};
};
 
struct basic_storage
{
    virtual ~basic_storage() = default;
 
    virtual void unload_all(tpp::thread_pool& pool) = 0;
 
    virtual void unload_single(tpp::thread_pool& pool, const std::string& key) = 0;
 
    virtual void unload_group(tpp::thread_pool& pool, const std::string& group) = 0;
 
    virtual void evict_unused(tpp::thread_pool& pool, const std::string& group, std::chrono::steady_clock::duration max_idle) = 0;
 
    virtual void preload_all() = 0;
};
 
template<typename T>
struct asset_storage : public basic_storage
{
    using request_container_t = std::unordered_map<std::string, asset_handle<T>>;
    template<typename F>
    using callable = std::function<F>;
 
    using load_from_file_t = callable<bool(tpp::thread_pool& pool, asset_handle<T>&, const std::string&, load_mode)>;
 
    using predicate_t = callable<bool(const asset_handle<T>&)>;
    using load_from_instance_t = callable<bool(tpp::thread_pool& pool, asset_handle<T>&, std::shared_ptr<T>)>;
 
    ~asset_storage() override = default;
 
    void unload_handle(tpp::thread_pool& pool, asset_handle<T>& handle)
    {
        pool.stop(handle.task_id());
        handle.invalidate();
    }
 
    void unload_with_condition(tpp::thread_pool& pool, const predicate_t& predicate)
    {
        std::lock_guard<std::recursive_mutex> lock(container_mutex);
        for(auto it = container.begin(); it != container.end();)
        {
            if(predicate(it->second))
            {
                auto& handle = it->second;
                unload_handle(pool, handle);
                it = container.erase(it);
            }
            else
            {
                ++it;
            }
        }
    }
 
    void unload_all(tpp::thread_pool& pool) final
    {
        unload_with_condition(pool,
                              [](const auto& it)
                              {
                                  return true;
                              });
    }
 
    void unload_group(tpp::thread_pool& pool, const std::string& group) final
    {
        unload_with_condition(pool,
                              [&](const auto& it)
                              {
                                  const auto& id = it.id();
                                  hpp::string_view id_view(id);
                                  return id_view.starts_with(group);
                              });
    }
 
    void unload_single(tpp::thread_pool& pool, const std::string& key) final
    {
        unload_with_condition(pool,
                              [&](const auto& it)
                              {
                                  const auto& id = it.id();
                                  return id == key;
                              });
    }
 
    auto get_with_condition(const predicate_t& predicate) const -> std::vector<asset_handle<T>>
    {
        std::lock_guard<std::recursive_mutex> lock(container_mutex);
        std::vector<asset_handle<T>> result;
        result.reserve(container.size() + 1);
        result.emplace_back(asset_handle<T>::get_empty());
 
        for(const auto& kvp : container)
        {
            if(predicate(kvp.second))
            {
                result.emplace_back(kvp.second);
            }
        }
 
        return result;
    }
 
    auto get_group(const std::string& group) const -> std::vector<asset_handle<T>>
    {
        return get_with_condition(
            [&](const auto& it)
            {
                const auto& id = it.id();
                hpp::string_view id_view(id);
                return id_view.starts_with(group);
            });
    }
 
    void evict_unused(tpp::thread_pool& pool, const std::string& group, std::chrono::steady_clock::duration max_idle) final
    {
        auto now = std::chrono::steady_clock::now();
        std::lock_guard<std::recursive_mutex> lock(container_mutex);
        for(auto& [key, handle] : container)
        {
            const auto& id = handle.id();
            hpp::string_view id_view(id);
            if(!id_view.starts_with(group))
            {
                continue;
            }
            if(!handle.is_ready())
            {
                continue;
            }
            bool is_idle = (now - handle.last_access()) > max_idle;
            if(!is_idle)
            {
                continue;
            }
            pool.stop(handle.task_id());
            handle.demote_to_deferred();
            load_from_file(pool, handle, key, load_mode::deferred);
        }
    }
 
    void preload_all() final
    {
        std::lock_guard<std::recursive_mutex> lock(container_mutex);
        for(auto& [key, handle] : container)
        {
            if(handle.is_deferred())
            {
                handle.submit();
            }
        }
    }
 
    load_from_file_t load_from_file;
    load_from_instance_t load_from_instance;
    request_container_t container;
    mutable std::recursive_mutex container_mutex;
};
 
} // namespace unravel