Files
multi_render_blender/src/main/QueueManager.js

652 lines
26 KiB
JavaScript

const BlenderProcess = require("./BlenderProcess.js");
const BlenderDaemon = require("./BlenderDaemon.js");
const path = require("path");
const fs = require("fs");
const os = require("os");
const { Notification } = require("electron");
const STR_STATUS_IDLE = "idle";
const STR_STATUS_RUNNING = "running";
const STR_STATUS_PAUSED = "paused";
const NB_PLACEHOLDER_MAX_SIZE = 512;
class QueueManager {
constructor(obj_window) {
this.obj_window = obj_window;
this.list_queue = [];
this.nb_current_index = 0;
this.str_status = STR_STATUS_IDLE;
this.obj_current_process = null;
this.nb_last_render_ms = 0;
this.str_last_image_path = null;
this.obj_notification_config = { is_notify_each_image: false, is_notify_all_done: true };
this.str_hostname = os.hostname();
this.nb_total_render_ms = 0;
this.nb_completed_renders = 0;
this.map_remote_avgs = {};
this.obj_email_notifier = null;
}
set_notification_config(obj_config) {
this.obj_notification_config = obj_config || { is_notify_each_image: false, is_notify_all_done: true };
}
set_email_notifier(obj_notifier) {
this.obj_email_notifier = obj_notifier;
}
start(obj_config) {
if (this.str_status === STR_STATUS_PAUSED) {
this.str_status = STR_STATUS_RUNNING;
this._send_log("Reprise du rendu...");
this._process_next();
return Promise.resolve({ is_success: true });
}
this.list_queue = this._build_queue(obj_config);
this.nb_current_index = 0;
this.str_status = STR_STATUS_RUNNING;
this.nb_last_render_ms = 0;
this.str_last_image_path = null;
this.nb_total_render_ms = 0;
this.nb_completed_renders = 0;
this.map_remote_avgs = {};
this.str_overwrite_mode = obj_config.str_overwrite_mode || "overwrite";
this.list_collections = obj_config.list_collections || [];
this.obj_render_settings = obj_config.obj_render_settings || null;
if (this.str_overwrite_mode === "skip") {
this._prescan_existing();
}
this._send_log("File de rendu construite : " + this.list_queue.length + " elements.");
this._send_progress();
this._send_log("Demarrage du daemon Blender...");
BlenderDaemon.start(obj_config.str_blend_file)
.then(() => {
this._send_log("Daemon Blender pret.");
this._process_next();
})
.catch((obj_err) => {
this._send_log("ERREUR demarrage daemon : " + (obj_err.message || String(obj_err)));
this.str_status = STR_STATUS_IDLE;
this._send_progress();
});
return Promise.resolve({ is_success: true, nb_total: this.list_queue.length });
}
pause() {
if (this.str_status !== STR_STATUS_RUNNING) {
return Promise.resolve({ is_success: false });
}
this.str_status = STR_STATUS_PAUSED;
this._send_log("Rendu en pause.");
return Promise.resolve({ is_success: true });
}
stop() {
let str_file_to_delete = null;
if (this.nb_current_index < this.list_queue.length) {
let obj_item = this.list_queue[this.nb_current_index];
if (obj_item.str_status === "rendering") {
obj_item.str_status = "stopped";
str_file_to_delete = obj_item.str_expected_file;
}
}
this.str_status = STR_STATUS_IDLE;
BlenderDaemon.stop();
if (this.obj_current_process) {
this.obj_current_process.kill("SIGTERM");
this.obj_current_process = null;
}
if (str_file_to_delete) {
try {
if (fs.existsSync(str_file_to_delete)) {
fs.unlinkSync(str_file_to_delete);
this._send_log("Fichier partiel supprime : " + path.basename(str_file_to_delete));
}
} catch (obj_del_err) {
this._send_log("Erreur suppression fichier partiel : " + obj_del_err.message);
}
}
this._send_log("Rendu arrete.");
this._send_progress();
return Promise.resolve({ is_success: true });
}
check_queue(obj_config) {
let list_queue = this._build_queue(obj_config);
let list_existing = [];
for (let nb_i = 0; nb_i < list_queue.length; nb_i++) {
let obj_item = list_queue[nb_i];
try {
if (fs.existsSync(obj_item.str_expected_file)) {
let nb_size = fs.statSync(obj_item.str_expected_file).size;
if (nb_size > 0) {
list_existing.push({ nb_index: nb_i, str_path: obj_item.str_expected_file });
}
}
} catch (obj_err) {
// Fichier inaccessible, on ignore
}
}
return Promise.resolve({ list_existing: list_existing, nb_total: list_queue.length });
}
_build_queue(obj_config) {
let list_queue = [];
let str_blend_path = obj_config.str_blend_file;
let str_mode = obj_config.str_render_mode;
let str_base_output = obj_config.str_output_path;
let str_output_mode = obj_config.str_output_mode || "subfolder";
let str_frame_prefix = obj_config.str_frame_prefix !== undefined ? obj_config.str_frame_prefix : "f_";
let nb_frame_padding = obj_config.nb_frame_padding || 5;
let list_cameras = obj_config.list_cameras;
let list_enabled = [];
for (let obj_cam of list_cameras) {
if (obj_cam.is_enabled) {
list_enabled.push(obj_cam);
}
}
if (str_mode === "camera_by_camera") {
for (let obj_cam of list_enabled) {
let nb_step = obj_cam.nb_frame_step || 1;
for (let nb_frame = obj_cam.nb_frame_start; nb_frame <= obj_cam.nb_frame_end; nb_frame += nb_step) {
list_queue.push(this._create_queue_item(str_blend_path, str_base_output, str_output_mode, str_frame_prefix, nb_frame_padding, obj_cam, nb_frame));
}
}
} else {
let nb_min_frame = Infinity;
let nb_max_frame = -Infinity;
for (let obj_cam of list_enabled) {
if (obj_cam.nb_frame_start < nb_min_frame) {
nb_min_frame = obj_cam.nb_frame_start;
}
if (obj_cam.nb_frame_end > nb_max_frame) {
nb_max_frame = obj_cam.nb_frame_end;
}
}
for (let nb_frame = nb_min_frame; nb_frame <= nb_max_frame; nb_frame++) {
for (let obj_cam of list_enabled) {
let nb_cam_step = obj_cam.nb_frame_step || 1;
if (nb_frame >= obj_cam.nb_frame_start && nb_frame <= obj_cam.nb_frame_end && (nb_frame - obj_cam.nb_frame_start) % nb_cam_step === 0) {
list_queue.push(this._create_queue_item(str_blend_path, str_base_output, str_output_mode, str_frame_prefix, nb_frame_padding, obj_cam, nb_frame));
}
}
}
}
return list_queue;
}
_create_queue_item(str_blend_path, str_base_output, str_output_mode, str_frame_prefix, nb_frame_padding, obj_cam, nb_frame) {
let str_padded_frame = String(nb_frame).padStart(nb_frame_padding, "0");
let str_hash_pattern = "#".repeat(nb_frame_padding);
let str_ext = obj_cam.str_format.toLowerCase();
if (str_ext === "open_exr") {
str_ext = "exr";
} else if (str_ext === "jpeg") {
str_ext = "jpg";
} else if (str_ext === "tiff") {
str_ext = "tif";
}
let str_output_path = "";
let str_expected_file = "";
if (str_output_mode === "prefix") {
str_output_path = path.join(str_base_output, obj_cam.str_name + "_" + str_frame_prefix + str_hash_pattern);
str_expected_file = path.join(str_base_output, obj_cam.str_name + "_" + str_frame_prefix + str_padded_frame + "." + str_ext);
} else if (str_output_mode === "both") {
let str_output_dir = path.join(str_base_output, obj_cam.str_name);
str_output_path = path.join(str_output_dir, obj_cam.str_name + "_" + str_frame_prefix + str_hash_pattern);
str_expected_file = path.join(str_output_dir, obj_cam.str_name + "_" + str_frame_prefix + str_padded_frame + "." + str_ext);
} else {
let str_output_dir = path.join(str_base_output, obj_cam.str_name);
str_output_path = path.join(str_output_dir, str_frame_prefix + str_hash_pattern);
str_expected_file = path.join(str_output_dir, str_frame_prefix + str_padded_frame + "." + str_ext);
}
return {
str_blend_path: str_blend_path,
str_camera_name: obj_cam.str_name,
nb_frame: nb_frame,
nb_resolution_x: obj_cam.nb_resolution_x,
nb_resolution_y: obj_cam.nb_resolution_y,
str_format: obj_cam.str_format,
str_output_path: str_output_path,
str_expected_file: str_expected_file,
str_status: "pending",
};
}
_prescan_existing() {
let nb_skipped = 0;
let nb_remote = 0;
for (let obj_item of this.list_queue) {
try {
if (!fs.existsSync(obj_item.str_expected_file)) {
continue;
}
let nb_size = fs.statSync(obj_item.str_expected_file).size;
if (nb_size === 0) {
continue;
}
if (nb_size > NB_PLACEHOLDER_MAX_SIZE) {
obj_item.str_status = "skipped";
try {
obj_item.str_done_date = fs.statSync(obj_item.str_expected_file).mtime.toISOString();
} catch (obj_date_err) {
// ignore
}
nb_skipped++;
} else {
obj_item.str_status = "rendering_remote";
this._read_placeholder(obj_item);
nb_remote++;
}
} catch (obj_err) {
// Fichier inaccessible, on ignore
}
}
if (nb_skipped > 0 || nb_remote > 0) {
this._send_log("Pre-scan : " + nb_skipped + " fichier(s) existant(s), " + nb_remote + " en cours sur d'autres machines.");
}
}
_process_next() {
if (this.str_status !== STR_STATUS_RUNNING) {
return;
}
this._check_remote_completions();
// Batch skip : sauter les items deja marques par le prescan ou nouvellement detectes
let nb_skip_count = 0;
while (this.nb_current_index < this.list_queue.length && this.str_overwrite_mode === "skip") {
let obj_check = this.list_queue[this.nb_current_index];
// Item deja marque par le prescan
if (obj_check.str_status === "skipped" || obj_check.str_status === "rendering_remote") {
this.nb_current_index++;
nb_skip_count++;
continue;
}
// Verification fichier pour les items non pre-scannes (apparus entre-temps)
let is_exists = false;
let nb_size = 0;
try {
is_exists = fs.existsSync(obj_check.str_expected_file);
if (is_exists) {
nb_size = fs.statSync(obj_check.str_expected_file).size;
}
} catch (obj_fs_err) {
is_exists = false;
}
if (!is_exists) {
break;
}
if (nb_size === 0) {
break;
}
if (nb_size > NB_PLACEHOLDER_MAX_SIZE) {
obj_check.str_status = "skipped";
try {
obj_check.str_done_date = fs.statSync(obj_check.str_expected_file).mtime.toISOString();
} catch (obj_date_err) {
// ignore
}
} else {
obj_check.str_status = "rendering_remote";
this._read_placeholder(obj_check);
}
this.nb_current_index++;
nb_skip_count++;
}
if (nb_skip_count > 0) {
this._send_log("Skip : " + nb_skip_count + " fichier(s) existant(s)");
this._send_progress();
}
if (this.nb_current_index >= this.list_queue.length) {
this.str_status = STR_STATUS_IDLE;
BlenderDaemon.stop();
this._send_log("Tous les rendus sont termines !");
this._send_event("render-complete", { is_all_done: true });
if (this.obj_notification_config.is_notify_all_done) {
this._send_notification("Rendus termines", "Tous les rendus de la file sont termines (" + this.list_queue.length + " elements).");
}
return;
}
let obj_item = this.list_queue[this.nb_current_index];
// Re-verification avant rendu : rattrape les fichiers non detectes par le batch skip
if (this.str_overwrite_mode === "skip") {
try {
if (fs.existsSync(obj_item.str_expected_file)) {
let nb_recheck_size = fs.statSync(obj_item.str_expected_file).size;
if (nb_recheck_size > NB_PLACEHOLDER_MAX_SIZE) {
obj_item.str_status = "skipped";
try {
obj_item.str_done_date = fs.statSync(obj_item.str_expected_file).mtime.toISOString();
} catch (obj_d_err) {
// ignore
}
this._send_log("Skip : " + obj_item.str_camera_name + " F" + obj_item.nb_frame + " (existant)");
this.nb_current_index++;
this._send_progress();
this._process_next();
return;
} else if (nb_recheck_size > 0) {
obj_item.str_status = "rendering_remote";
this._read_placeholder(obj_item);
this._send_log("Skip : " + obj_item.str_camera_name + " F" + obj_item.nb_frame + " (en cours par " + (obj_item.str_remote_hostname || "?") + ")");
this.nb_current_index++;
this._send_progress();
this._process_next();
return;
}
}
} catch (obj_recheck_err) {
// Erreur fs, on continue le rendu
}
}
obj_item.str_status = "rendering";
if (this.str_overwrite_mode === "skip") {
try {
let str_dir = path.dirname(obj_item.str_expected_file);
if (!fs.existsSync(str_dir)) {
fs.mkdirSync(str_dir, { recursive: true });
}
fs.writeFileSync(obj_item.str_expected_file, this._get_placeholder_content());
} catch (obj_file_err) {
this._send_log("ERREUR creation placeholder : " + obj_file_err.message);
}
}
this._send_log("Rendu : " + obj_item.str_camera_name + " - Frame " + obj_item.nb_frame);
this._send_progress();
let nb_start = Date.now();
BlenderDaemon.render_frame({
str_camera_name: obj_item.str_camera_name,
nb_frame: obj_item.nb_frame,
nb_resolution_x: obj_item.nb_resolution_x,
nb_resolution_y: obj_item.nb_resolution_y,
str_format: obj_item.str_format,
str_output_path: obj_item.str_expected_file,
list_collections: this.list_collections,
obj_render_settings: this.obj_render_settings,
fn_on_stdout: (str_data) => {
this._send_log(str_data.trim());
},
})
.then((obj_result) => {
this.obj_current_process = null;
if (this.str_status !== STR_STATUS_RUNNING) {
return;
}
obj_item.str_status = "done";
this.nb_last_render_ms = Date.now() - nb_start;
this.nb_total_render_ms += this.nb_last_render_ms;
this.nb_completed_renders++;
obj_item.str_done_date = new Date().toISOString();
let str_image = obj_result.str_rendered_file || obj_item.str_expected_file;
this.str_last_image_path = str_image;
this._send_event("preview-update", str_image);
this._send_log("Termine : " + str_image);
if (this.obj_notification_config.is_notify_each_image) {
this._send_notification("Rendu termine", obj_item.str_camera_name + " — Frame " + obj_item.nb_frame);
}
this.nb_current_index++;
this._send_progress();
this._process_next();
})
.catch((obj_err) => {
this.obj_current_process = null;
if (this.str_overwrite_mode === "skip" && fs.existsSync(obj_item.str_expected_file)) {
try {
let obj_stats = fs.statSync(obj_item.str_expected_file);
if (obj_stats.size <= NB_PLACEHOLDER_MAX_SIZE) {
fs.unlinkSync(obj_item.str_expected_file);
}
} catch (obj_cleanup_err) {
// Ignore cleanup errors
}
}
if (this.str_status !== STR_STATUS_RUNNING) {
return;
}
obj_item.str_status = "error";
this.nb_last_render_ms = Date.now() - nb_start;
this.str_last_image_path = null;
let str_err_msg = obj_err.str_message || obj_err.message || String(obj_err);
this._send_log("ERREUR : " + str_err_msg);
this._send_event("render-error", {
str_camera: obj_item.str_camera_name,
nb_frame: obj_item.nb_frame,
str_error: str_err_msg,
});
this.nb_current_index++;
this._send_progress();
let is_gpu_error = str_err_msg.indexOf("CUDA") !== -1
|| str_err_msg.indexOf("GPU memory") !== -1
|| str_err_msg.indexOf("out of memory") !== -1
|| str_err_msg.indexOf("OpenCL") !== -1
|| str_err_msg.indexOf("HIP") !== -1;
if (!BlenderDaemon.is_running() || is_gpu_error) {
if (is_gpu_error) {
this._send_log("Erreur GPU detectee, redemarrage du daemon pour contexte CUDA neuf...");
BlenderDaemon.stop();
} else {
this._send_log("Daemon crash detecte, redemarrage...");
}
BlenderDaemon.start(obj_item.str_blend_path)
.then(() => {
this._send_log("Daemon redemarre.");
this._process_next();
})
.catch((obj_restart_err) => {
this._send_log("ERREUR redemarrage daemon : " + (obj_restart_err.message || String(obj_restart_err)));
this.str_status = STR_STATUS_IDLE;
this._send_progress();
});
} else {
this._process_next();
}
});
}
_send_progress() {
let obj_item = this.list_queue[this.nb_current_index] || {};
let list_skipped = [];
let list_stopped = [];
let list_skipped_paths = [];
let list_rendering_remote = [];
let list_item_results = [];
let nb_avg = this.nb_completed_renders > 0
? Math.round(this.nb_total_render_ms / this.nb_completed_renders)
: 0;
for (let nb_i = 0; nb_i < this.list_queue.length; nb_i++) {
let obj_q = this.list_queue[nb_i];
if (obj_q.str_status === "skipped") {
list_skipped.push(nb_i);
list_skipped_paths.push({ nb_index: nb_i, str_path: obj_q.str_expected_file });
list_item_results.push({
nb_index: nb_i,
str_type: "done",
str_date: obj_q.str_done_date || null,
str_resolution: obj_q.nb_resolution_x + "x" + obj_q.nb_resolution_y,
});
} else if (obj_q.str_status === "stopped") {
list_stopped.push(nb_i);
} else if (obj_q.str_status === "rendering_remote") {
list_rendering_remote.push(nb_i);
list_item_results.push({
nb_index: nb_i,
str_type: "rendering_remote",
str_remote_hostname: obj_q.str_remote_hostname || "?",
nb_remote_avg_ms: obj_q.nb_remote_avg_ms || 0,
});
} else if (obj_q.str_status === "done") {
list_item_results.push({
nb_index: nb_i,
str_type: "done",
str_date: obj_q.str_done_date || null,
str_resolution: obj_q.nb_resolution_x + "x" + obj_q.nb_resolution_y,
});
}
}
let list_machine_avgs = [];
if (nb_avg > 0) {
list_machine_avgs.push({ str_hostname: this.str_hostname, nb_avg_ms: nb_avg });
}
for (let str_h of Object.keys(this.map_remote_avgs)) {
if (this.map_remote_avgs[str_h] > 0) {
list_machine_avgs.push({ str_hostname: str_h, nb_avg_ms: this.map_remote_avgs[str_h] });
}
}
let nb_remaining = 0;
for (let nb_r = this.nb_current_index; nb_r < this.list_queue.length; nb_r++) {
let obj_r = this.list_queue[nb_r];
if (obj_r.str_status === "pending" || obj_r.str_status === "rendering") {
nb_remaining++;
}
}
this._send_event("render-progress", {
nb_current: this.nb_current_index,
nb_total: this.list_queue.length,
nb_remaining: nb_remaining,
str_camera: obj_item.str_camera_name || "-",
nb_frame: obj_item.nb_frame || 0,
str_status: this.str_status,
nb_last_render_ms: this.nb_last_render_ms,
str_last_image_path: this.str_last_image_path,
list_skipped: list_skipped,
list_stopped: list_stopped,
list_skipped_paths: list_skipped_paths,
list_rendering_remote: list_rendering_remote,
list_item_results: list_item_results,
str_hostname: this.str_hostname,
nb_avg_render_ms: nb_avg,
list_machine_avgs: list_machine_avgs,
});
}
_get_placeholder_content() {
let nb_avg = this.nb_completed_renders > 0
? Math.round(this.nb_total_render_ms / this.nb_completed_renders)
: 0;
return JSON.stringify({ str_hostname: this.str_hostname, nb_avg_ms: nb_avg });
}
_read_placeholder(obj_item) {
try {
let str_content = fs.readFileSync(obj_item.str_expected_file, "utf-8");
let obj_data = JSON.parse(str_content);
obj_item.str_remote_hostname = obj_data.str_hostname || "?";
obj_item.nb_remote_avg_ms = obj_data.nb_avg_ms || 0;
if (obj_data.str_hostname && obj_data.nb_avg_ms > 0) {
this.map_remote_avgs[obj_data.str_hostname] = obj_data.nb_avg_ms;
}
} catch (obj_parse_err) {
obj_item.str_remote_hostname = "?";
obj_item.nb_remote_avg_ms = 0;
}
}
_check_remote_completions() {
for (let obj_q of this.list_queue) {
if (obj_q.str_status !== "rendering_remote") {
continue;
}
try {
if (fs.existsSync(obj_q.str_expected_file)) {
let nb_size = fs.statSync(obj_q.str_expected_file).size;
if (nb_size > NB_PLACEHOLDER_MAX_SIZE) {
obj_q.str_status = "skipped";
obj_q.str_done_date = fs.statSync(obj_q.str_expected_file).mtime.toISOString();
}
} else {
obj_q.str_status = "pending";
}
} catch (obj_check_err) {
// ignore
}
}
}
_send_log(str_message) {
this._send_event("log", str_message);
}
_send_event(str_channel, obj_data) {
if (this.obj_window && !this.obj_window.isDestroyed()) {
this.obj_window.webContents.send(str_channel, obj_data);
}
}
_send_notification(str_title, str_body) {
if (Notification.isSupported()) {
let obj_notif = new Notification({ title: str_title, body: str_body });
obj_notif.show();
}
if (this.obj_email_notifier) {
this.obj_email_notifier.send("[Multi Render] " + str_title, str_body)
.then((obj_result) => {
if (obj_result.is_success) {
this._send_log("Email envoye : " + str_title);
} else if (obj_result.str_error && obj_result.str_error !== "Email non configure") {
this._send_log("Erreur email : " + obj_result.str_error);
}
})
.catch((obj_err) => {
this._send_log("Erreur email : " + obj_err.message);
});
}
}
}
module.exports = QueueManager;