39using namespace std::literals;
48auto checking_dependencies_job_name() -> std::string
54auto needs_recompilation(
const fs::path& source_file_path,
const fs::path& compiled_output_path) ->
bool
59 if(!fs::exists(compiled_output_path, err) || err)
66 if(!fs::exists(manifest_path, err) || err)
74 asset_compiler::asset_manifest manifest;
77 APPLOG_TRACE(
"Failed to load manifest for {}, recompilation needed", compiled_output_path.string());
84 APPLOG_TRACE(
"Source file changed for {}, recompilation needed", compiled_output_path.string());
93void resolve_includes(
const fs::path& file_path, std::set<fs::path>& processed_files)
98void resolve_includes<gfx::shader>(
const fs::path& file_path, std::set<fs::path>& processed_files)
100 if(!processed_files.insert(file_path).second)
105 std::ifstream file(file_path);
112 const std::string include_keyword =
"#include";
113 const std::string bgfx_include_path =
"/path/to/bgfx/include";
116 while(std::getline(file, line))
119 line.erase(0, line.find_first_not_of(
" \t"));
122 if(line.compare(0, include_keyword.length(), include_keyword) == 0)
125 size_t start = line.find_first_of(
"\"<") + 1;
126 size_t end = line.find_last_of(
"\">");
128 if(start == std::string::npos || end == std::string::npos || start >= end)
134 std::string include_path = line.substr(start, end - start);
135 fs::path resolved_path;
137 if(line[start - 1] ==
'<' && line[end] ==
'>')
146 resolved_path = file_path.parent_path() / include_path;
149 resolved_path = fs::absolute(resolved_path);
153 resolve_includes<gfx::shader>(resolved_path, processed_files);
158auto extract_href_from_link_tag(hpp::string_view link_tag) -> hpp::string_view
160 size_t href_pos = link_tag.find(
"href=");
161 if(href_pos == std::string::npos)
169 char quote_char = link_tag[href_pos];
170 if(quote_char !=
'"' && quote_char !=
'\'')
176 size_t href_end = link_tag.find(quote_char, href_pos);
177 if(href_end == std::string::npos)
182 return link_tag.substr(href_pos, href_end - href_pos);
185auto resolve_ui_tree_dependency_path(
const fs::path& href_value,
const fs::path& base_file_path) -> fs::path
194 return fs::absolute(base_file_path.parent_path() / href_value);
198void resolve_includes<ui_tree>(
const fs::path& file_path, std::set<fs::path>& processed_files)
200 if(!processed_files.insert(file_path).second)
205 std::ifstream file(file_path);
211 std::string content_str((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
212 hpp::string_view content(content_str);
215 while((pos = content.find(
"<link", pos)) != std::string::npos)
218 size_t tag_end = content.find(
'>', pos);
219 if(tag_end == std::string::npos)
224 hpp::string_view link_tag = content.substr(pos, tag_end - pos + 1);
225 hpp::string_view href_value = extract_href_from_link_tag(link_tag);
227 if(!href_value.empty())
229 fs::path resolved_path = resolve_ui_tree_dependency_path(fs::path(href_value), file_path);
233 auto extension = resolved_path.extension().string();
234 if(std::find(supported_deps.begin(), supported_deps.end(), extension) != supported_deps.end())
236 resolve_includes<ui_tree>(resolved_path, processed_files);
245auto has_depencency(
const fs::path& file,
const fs::path& dep_to_check) ->
bool
247 std::set<fs::path> dependecies;
248 resolve_includes<T>(file, dependecies);
250 return dependecies.contains(dep_to_check);
253auto remove_meta_tag(
const fs::path& synced_path) -> fs::path
258auto remove_meta_tag(
const std::vector<fs::path>& synced_paths) -> std::vector<fs::path>
260 std::decay_t<
decltype(synced_paths)> reduced;
261 reduced.reserve(synced_paths.size());
262 for(
const auto& synced_path : synced_paths)
264 reduced.emplace_back(remove_meta_tag(synced_path));
269void unwatch(std::vector<uint64_t>& watchers)
271 for(
const auto&
id : watchers)
278auto get_asset_key(
const fs::path& path) -> std::string
287auto get_meta_key(
const fs::path& path) -> std::string
293 return key +
".meta";
296auto check_files_integrity(
const std::string& key,
const fs::path& entry_path) ->
bool
301 if(!fs::exists(key_path, ec))
304 fs::remove(entry_path, ec);
306 auto meta = get_meta_key(entry_path);
308 if(fs::exists(meta_path, ec))
311 fs::remove(meta_path, ec);
324 auto& am = ctx.get_cached<asset_manager>();
325 auto& ts = ctx.get_cached<threader>();
326 auto& tm = ctx.get_cached<thumbnail_manager>();
327 auto& em = ctx.get_cached<editing_manager>();
329 fs::path watch_dir = fs::path(dir).make_preferred();
331 auto callback = [&am, &ts, &tm, &em](
const auto& entries,
bool is_initial_list)
333 std::set<hpp::uuid> changed;
334 std::set<hpp::uuid> removed;
336 for(
const auto&
entry : entries)
338 auto key = get_asset_key(
entry.path);
342 if(
entry.type == fs::file_type::regular)
346 removed.emplace(am.get_asset<T>(key).uid());
347 am.unload_asset<T>(
key);
349 if constexpr(std::is_same<T, script>::value)
356 auto old_key = get_asset_key(
entry.last_path);
357 am.rename_asset<T>(old_key,
key);
359 if constexpr(std::is_same<T, script>::value)
366 if(check_files_integrity(key,
entry.path))
369 auto asset = am.get_asset<T>(
key, flags);
371 changed.emplace(asset.uid());
374 if constexpr(std::is_same<T, script>::value)
382 if(!changed.empty() || !removed.empty())
384 tpp::invoke(tpp::main_thread::get_id(),
385 [&tm, &em, &am, changed, removed]()
387 for(
const auto& uid : removed)
389 tm.remove_thumbnail(uid);
392 for(
const auto& uid : changed)
394 tm.regenerate_thumbnail(uid);
396 if constexpr(std::is_same<T, prefab>::value)
398 auto asset = am.get_asset<T>(uid);
399 em.on_prefab_updated(asset);
412 auto& am = ctx.get_cached<asset_manager>();
413 auto& ts = ctx.get_cached<threader>();
415 fs::path watch_dir = fs::path(dir).make_preferred();
417 auto callback = [&am, &ts](
const auto& entries,
bool is_initial_list)
424 for(
const auto&
entry : entries)
428 if(
entry.type == fs::file_type::regular)
438 auto task = ts.pool->schedule(checking_dependencies_job_name<T>(),
441 auto assets = am.get_assets<T>();
442 for(
const auto& asset : assets)
444 auto meta = am.get_metadata(asset.uid());
447 if(has_depencency<T>(absolute_path,
entry.path))
471 [&ts, &am](
const std::string& ext,
const auto& ref_path,
const auto& synced_paths,
bool is_initial_listing)
473 auto paths = remove_meta_tag(synced_paths);
475 for(
const auto& output : paths)
478 if(is_initial_listing && !needs_recompilation(ref_path, output))
483 auto key = get_asset_key(output);
484 if(check_files_integrity(key, output))
486 auto task = ts.pool->schedule(get_job_name<T>(),
487 [&am, ref_path, output]()
507static void watch_synced(
rtti::context& ctx, std::vector<uint64_t>& watchers,
const fs::path& dir)
512 watchers.push_back(watch_id);
526 [&ts, &am](
const std::string& ext,
const auto& ref_path,
const auto& synced_paths,
bool is_initial_listing)
528 auto paths = remove_meta_tag(synced_paths);
535 for(
const auto& output : paths)
538 std::find(std::begin(platform_supported), std::end(platform_supported), output.extension().string());
540 if(it == std::end(platform_supported))
546 if(is_initial_listing && !needs_recompilation(ref_path, output))
551 auto key = get_asset_key(output);
552 if(check_files_integrity(key, output))
554 auto task = ts.pool->schedule(get_job_name<gfx::shader>(),
555 [&am, ref_path, output]()
578void watch_synced<gfx::shader>(
rtti::context& ctx, std::vector<uint64_t>& watchers,
const fs::path& dir)
583 const auto watch_id =
584 watch_assets<gfx::shader>(ctx, dir,
fs::pattern_filter(
"*" +
type +
".asset" + renderer_extension),
true);
585 watchers.push_back(watch_id);
593 const auto on_dir_modified =
594 [](
const std::string& ext,
const auto& ,
const auto& ,
bool )
598 const auto on_dir_removed = [](
const std::string& ext,
const auto& ,
const auto& synced_paths)
600 for(
const auto& synced_path : synced_paths)
603 fs::remove_all(synced_path, err);
607 const auto on_dir_renamed = [](
const std::string& ext,
const auto& ,
const auto& synced_paths)
609 for(
const auto& synced_path : synced_paths)
612 fs::rename(synced_path.first, synced_path.second, err);
619 std::vector<uint64_t>& watchers,
621 const fs::path& data_dir,
622 const fs::path& meta_dir,
625 setup_directory(ctx, syncer);
628 const auto on_file_removed = [&am](
const std::string& ext,
const auto& ref_path,
const auto& synced_paths)
630 for(
const auto& synced_path : synced_paths)
633 fs::remove_all(synced_path, err);
635 am.remove_asset_info_for_path(ref_path);
639 const auto on_file_renamed = [](
const std::string& ext,
const auto& ,
const auto& synced_paths)
641 for(
const auto& synced_path : synced_paths)
644 fs::rename(synced_path.first, synced_path.second, err);
648 const auto on_file_modified =
649 [&am](
const std::string& ext,
const auto& ref_path,
const auto& synced_paths,
bool is_initial_listing)
651 for(
const auto& synced_path : synced_paths)
655 if(fs::exists(synced_path, err))
660 if(meta.uid.is_nil())
663 meta = am.generate_metadata(key);
665 meta.uid = am.add_asset_info_for_path(ref_path, meta,
true);
673 for(
const auto&
type : asset_set)
675 syncer.
set_mapping(
type, {
".meta"}, on_file_modified, on_file_modified, on_file_removed, on_file_renamed);
681 auto id = watch_assets_depenencies<gfx::shader>(ctx, data_dir,
fs::pattern_filter(
"*" + dep_ex));
682 watchers.emplace_back(
id);
687 auto id = watch_assets_depenencies<ui_tree>(ctx, data_dir,
fs::pattern_filter(
"*" + dep_ex));
688 watchers.emplace_back(
id);
691 syncer.
sync(data_dir, meta_dir);
701 std::vector<uint64_t>& watchers,
703 const fs::path& meta_dir,
704 const fs::path& cache_dir,
707 setup_directory(ctx, syncer);
709 auto on_removed = [](
const std::string& ext,
const auto& ,
const auto& synced_paths)
711 for(
const auto& synced_path : synced_paths)
713 auto synced_asset = remove_meta_tag(synced_path);
717 fs::remove_all(manifest_path, err);
718 fs::remove_all(synced_asset, err);
722 auto on_renamed = [](
const std::string& ext,
const auto& ,
const auto& synced_paths)
724 for(
const auto& synced_path : synced_paths)
726 auto synced_old_asset = remove_meta_tag(synced_path.first);
727 auto synced_new_asset = remove_meta_tag(synced_path.second);
731 fs::rename(manifest_old_path, manifest_new_path, err);
732 fs::rename(synced_old_asset, synced_new_asset, err);
736 add_to_syncer<gfx::texture>(ctx, syncer, on_removed, on_renamed);
737 add_to_syncer<gfx::shader>(ctx, syncer, on_removed, on_renamed);
738 add_to_syncer<mesh>(ctx, syncer, on_removed, on_renamed);
739 add_to_syncer<material>(ctx, syncer, on_removed, on_renamed);
740 add_to_syncer<animation_clip>(ctx, syncer, on_removed, on_renamed);
741 add_to_syncer<prefab>(ctx, syncer, on_removed, on_renamed);
742 add_to_syncer<scene_prefab>(ctx, syncer, on_removed, on_renamed);
743 add_to_syncer<physics_material>(ctx, syncer, on_removed, on_renamed);
744 add_to_syncer<audio_clip>(ctx, syncer, on_removed, on_renamed);
745 add_to_syncer<font>(ctx, syncer, on_removed, on_renamed);
746 add_to_syncer<script>(ctx, syncer, on_removed, on_renamed);
747 add_to_syncer<ui_tree>(ctx, syncer, on_removed, on_renamed);
748 add_to_syncer<style_sheet>(ctx, syncer, on_removed, on_renamed);
750 syncer.
sync(meta_dir, cache_dir);
758 watch_synced<gfx::texture>(ctx, watchers, cache_dir);
759 watch_synced<gfx::shader>(ctx, watchers, cache_dir);
760 watch_synced<mesh>(ctx, watchers, cache_dir);
761 watch_synced<material>(ctx, watchers, cache_dir);
762 watch_synced<animation_clip>(ctx, watchers, cache_dir);
763 watch_synced<prefab>(ctx, watchers, cache_dir);
764 watch_synced<scene_prefab>(ctx, watchers, cache_dir);
765 watch_synced<physics_material>(ctx, watchers, cache_dir);
766 watch_synced<audio_clip>(ctx, watchers, cache_dir);
767 watch_synced<font>(ctx, watchers, cache_dir);
768 watch_synced<script>(ctx, watchers, cache_dir);
769 watch_synced<ui_tree>(ctx, watchers, cache_dir);
770 watch_synced<style_sheet>(ctx, watchers, cache_dir);
781void asset_watcher::on_os_event(
rtti::context& ctx, os::event& e)
783 if(e.type == os::events::window)
785 if(e.window.type == os::window_event_id::focus_lost)
787 if(!os::window::is_any_focused())
793 if(
e.window.type == os::window_event_id::focus_gained)
795 if(os::window::is_any_focused())
806 APPLOG_TRACE(
"{}::{}", hpp::type_name_str(*
this), __func__);
809 ev.
on_os_event.connect(sentinel_, 1000,
this, &asset_watcher::on_os_event);
811 watch_assets(ctx,
"engine:/",
true);
818 APPLOG_TRACE(
"{}::{}", hpp::type_name_str(*
this), __func__);
820 unwatch_assets(ctx,
"engine:/");
826 auto& w = watched_protocols_[protocol];
832 setup_meta_syncer(ctx,
839 setup_cache_syncer(ctx,
849 auto& w = watched_protocols_[protocol];
852 w.meta_syncer.unsync();
853 w.cache_syncer.unsync();
855 watched_protocols_.erase(protocol);
A filter that combines include and exclude patterns for file/directory filtering.
void set_mapping(const std::string &ref_ext, const std::vector< std::string > &synced_ext, on_entry_created_t on_entry_created, on_entry_modified_t on_entry_modified, on_entry_removed_t on_entry_removed, on_entry_renamed_t on_entry_renamed)
Remaps a specific extension of the reference directory to extensions of the synced directory....
std::function< void(const std::string &, const rename_pair_t &, const std::vector< rename_pair_t > &)> on_entry_renamed_t
void sync(const fs::path &reference_dir, const fs::path &synced_dir)
Start syncing the synced_dir with reference to the reference_dir i.e changes that occur in the refere...
void set_directory_mapping(on_entry_created_t on_entry_created, on_entry_modified_t on_entry_modified, on_entry_removed_t on_entry_removed, on_entry_renamed_t on_entry_renamed)
std::function< void(const std::string &, const fs::path &, const std::vector< fs::path > &)> on_entry_removed_t
static void unwatch(std::uint64_t key)
Un-watches a previously registered file or directory.
static void touch(const fs::path &path, bool recursive, fs::file_time_type time=fs::now())
Sets the last modification time of a file or directory. by default sets the time to the current time.
static auto watch(const fs::path &path, const pattern_filter &filter, bool recursive, bool initial_list, clock_t::duration poll_interval, notify_callback callback) -> std::uint64_t
Watches a file or directory for modification and call back the specified std::function....
Manages assets, including loading, unloading, and storage.
void unload_group(const std::string &group)
Unloads all assets in a specified group.
auto deinit(rtti::context &ctx) -> bool
void watch_assets(rtti::context &ctx, const std::string &protocol, bool wait=false)
auto init(rtti::context &ctx) -> bool
void unwatch_assets(rtti::context &ctx, const std::string &protocol)
#define APPLOG_WARNING(...)
#define APPLOG_TRACE(...)
auto get_suported_formats< gfx::shader >() -> const std::vector< std::string > &
auto get_all_formats() -> const std::vector< std::vector< std::string > > &
auto get_compiled_directory_no_slash(const std::string &prefix={}) -> std::string
auto get_data_directory(const std::string &prefix={}) -> std::string
auto get_type() -> const std::string &
auto get_meta_directory_no_slash(const std::string &prefix={}) -> std::string
auto get_compiled_directory(const std::string &prefix={}) -> std::string
auto get_suported_dependencies_formats< gfx::shader >() -> const std::vector< std::string > &
auto get_meta_directory(const std::string &prefix={}) -> std::string
auto get_suported_formats() -> const std::vector< std::string > &
auto get_data_directory_no_slash(const std::string &prefix={}) -> std::string
auto get_suported_dependencies_formats() -> const std::vector< std::string > &
path extract_protocol(const path &_path)
Given the specified path/filename, resolve the final full filename. This will be based on either the ...
path reduce_trailing_extensions(const path &_path)
another.
path resolve_protocol(const path &_path)
Given the specified path/filename, resolve the final full filename. This will be based on either the ...
bool has_known_protocol(const path &_path)
Checks whether the path has a known protocol.
path replace(const path &_path, const path &_sequence, const path &_new_sequence)
Replacing any occurences of the specified path sequence with another.
auto to_string(const watcher::entry &e) -> std::string
path convert_to_protocol(const path &_path)
Oposite of the resolve_protocol this function tries to convert to protocol path from an absolute one.
auto get_renderer_platform_supported_filename_extensions() -> const std::vector< std::string > &
auto get_current_renderer_filename_extension() -> const std::string &
void end(encoder *_encoder)
auto start(seq_action action, const seq_scope_policy &scope_policy, hpp::source_location location) -> seq_id_t
Starts a new action.
auto get_manifest_path(const fs::path &compiled_asset_path) -> fs::path
Generate manifest file path from compiled asset path.
auto compile< gfx::shader >(asset_manager &am, const fs::path &key, const fs::path &output, uint32_t flags) -> bool
auto is_source_file_changed(const fs::path &source_path, const asset_manifest &manifest) -> bool
Check if source file has changed compared to manifest.
auto load_manifest(const fs::path &manifest_path, asset_manifest &manifest) -> bool
Load manifest from file.
auto compile(asset_manager &am, const fs::path &key, const fs::path &output_key, uint32_t flags=0) -> bool
auto get_job_name() -> std::string
void load_from_file(const std::string &absolute_path, animation_clip &obj)
void save_to_file(const std::string &absolute_path, const animation_clip &obj)
hpp::event< void(rtti::context &, os::event &e)> on_os_event
os events
static void set_needs_recompile(const std::string &protocol, bool now=false)