import logging
import os
import sys
import time
from math import floor, sqrt, isnan
from pathlib import Path
from typing import Union
from enum import Enum
import os
os.environ['PYGAME_HIDE_SUPPORT_PROMPT'] = "hide"
import pygame
import numpy as np
import tqdm
from OpenGL.GL import GL_FRAMEBUFFER, GL_NONE, GL_RGB32F, GL_RGBA32F, glGetIntegerv, GL_COLOR_ATTACHMENT0, GL_COLOR_ATTACHMENT2, \
GL_COLOR_BUFFER_BIT, GL_FRAMEBUFFER, GL_NONE, GL_RGB32F, GL_RGBA32F, GL_VIEWPORT, \
glBindFramebuffer, glClear, glGetIntegerv, glUseProgram, GL_MAJOR_VERSION
from .gl_utils import load_program, load_shader_from_file, read_texture, total_textures_size, gl_clear_all_caches, estimate_best_window_size
from .glsimulation import FINISH, SHADER_PATH, STRIPES_ON_NAP, GLSimulation, log_mem_used, time_it
from .results_store import ResultsStore, SimulationEventType, ResultType
from .simple_simulation import SimulationDurationType, SimpleSimulation
from .loaders.simple_sim_loader import load_simple_sim_to_gpu
from .utils import EveryNSeconds, seconds_to_duration_str
[docs]
class GlWindowManagerInterface(Enum):
@classmethod
[docs]
def for_glfw( klass, glfw_window):
""" If one wants to call the swap buffer function in glfw,
one needs to have a reference to the window.
"""
wmi = GlWindowManagerInterface.GLFW
wmi.window = glfw_window
return wmi
if MEM_TRACKING:
from pympler import tracker
[docs]
MOSTLY_DRY_MESHES_THRESHOLD = 1e-5
[docs]
class SimulationRunner:
[docs]
record: bool # True if record simulation progress
def __init__(self, glsim, record_path: Union[Path, str]=Path("."),
early_out_delta_max=1e-6, record=True,
page_flip_func=pygame.display.flip,
enable_alpha_recording=False): # FIXME this is for backward compatibility
assert early_out_delta_max is None or early_out_delta_max > 0, \
f"The early out threshold must be strictly positive to be useful. You gave {early_out_delta_max}."
self._glsim = glsim
self._early_out_delta_max=early_out_delta_max
self.current_quantity = 0
self._enable_alpha_recording = enable_alpha_recording
if isinstance(record_path, str):
self.record_path = Path(record_path)
else:
self.record_path = record_path
# This is used mainly during the unit tests to avoid recording too much stuff.
self._record = record
# self._results_store is None if we don't want to record.
self._results_store = None
self._refresh_zoom_timer = EveryNSeconds(2)
self.quantities = np.array([]) # init like that to silence PyLance's warning
self.simulation_finished = False
# self.h_convergence = None
# self._last_h = None
self.last_recorded_time = 0
self.last_step_duration = None
# Full timer will be initialized at full_run UNLESS it was
# previously initialized by a restart from record.
self._full_timer = None
# delta represents the last difference measured on h,qx,qy from
# one frame to the other. It reminds how decision about early
# simulation stop, was made.
self._last_early_out_delta = None
self._zoom = 1
self._mean = 0
self.records = []
self._previous_results = None
self._still_simulation_count = 0
# When working with a synthetic scenario, that's the place
# where you can find the bathymetry update file...
# .npy is importnat because if you use np.save(...) it will
# prepend that suffix automatically :-)
self._bathy_file = self.record_path / "simul.top_update.npy"
if self._bathy_file.exists():
# Note the last modification time so that we can detect a
# new modification later on.
self._bathy_file_mtime = os.stat(self._bathy_file).st_mtime
else:
self._bathy_file_mtime = None
# Upon initialisation, the GL Viewport must be set to the
# needs of the user. We'll use it for the progress plotting.
self._gl_window_viewport = glGetIntegerv(GL_VIEWPORT)
self._gl_page_flip_func = page_flip_func
@property
[docs]
def early_out_threshold(self) -> float:
return self._early_out_delta_max
@classmethod
[docs]
def quick_run(self, sim:SimpleSimulation, record_path: Union[Path, str], refresh_view: float=0.1,
early_out_threshold = None,
gl_wmi: GlWindowManagerInterface = GlWindowManagerInterface.PYGAME ) -> ResultsStore:
""" Run a full simulation quickly. By quickly we mean a lot of the setup of the simulator
(such as OpenGL context) is done automatically. In particular a progress window is
displayed.
- `sim`: the simulation you want to run
- `record_path` : a directory where to record the results of the simulation. We
accept `str` as well as `Path`.
- `refresh_view` : how often the progress window is displyaed (every N seconds)
"""
assert isinstance(sim, SimpleSimulation)
assert isinstance(gl_wmi, GlWindowManagerInterface)
if isinstance(record_path, str):
record_path = Path(record_path)
# For the sake of backward compatibility we reuse any currently active OpenGL context.
gl_context_active = False
if gl_wmi == GlWindowManagerInterface.PYGAME:
import pygame
try:
pygame.display.Info()
gl_context_active = True
except:
pass
if not gl_context_active:
pygame.init()
nfo = pygame.display.Info()
window_width, window_height = estimate_best_window_size(nfo.current_w, nfo.current_h, sim.param_nx, sim.param_ny)
pygame.display.set_mode((window_width, window_height), pygame.OPENGL|pygame.DOUBLEBUF, vsync = 0)
page_flipper = pygame.display.flip
elif gl_wmi == GlWindowManagerInterface.GLFW:
import glfw
try:
glfw.get_video_mode( glfw.get_primary_monitor())
gl_context_active = True
except:
pass
if not gl_context_active:
import glfw # late binding to stay ligh on the dependencies
glfw.init()
vm = glfw.get_video_mode( glfw.get_primary_monitor())
window_width, window_height = estimate_best_window_size(vm.size.width, vm.size.height, sim.param_nx, sim.param_ny)
window = glfw.create_window(window_width, window_height, f"WolfGPU - glfw {gl_context_active}", None, None)
gl_wmi.window = window
glfw.make_context_current(window)
page_flipper = lambda : glfw.swap_buffers(window)
else:
page_flipper = lambda : glfw.swap_buffers(gl_wmi.window)
gpu_simulator:GLSimulation = load_simple_sim_to_gpu(sim)
simulation_runner = SimulationRunner(gpu_simulator,record=True,
record_path=record_path, early_out_delta_max=early_out_threshold,
page_flip_func=page_flipper)
simulation_runner.full_run(refresh_view=refresh_view)
# To avoid issues when having several simulations runs. But be aware
# that because of this, one cannot query the `sim` passed that point
# (since OpenGL stuff will be wiped out).
gpu_simulator.drop_gl_resources()
gl_clear_all_caches()
if not gl_context_active:
if gl_wmi == GlWindowManagerInterface.PYGAME:
pygame.quit()
elif gl_wmi == GlWindowManagerInterface.GLFW:
glfw.destroy_window(gl_wmi.window)
glfw.terminate()
return ResultsStore(simulation_runner.results_path(), "r")
[docs]
def scanned_file(self):
return self._bathy_file
@property
[docs]
def step_num(self):
""" The step that is going to be computed. Starts at zero.
"""
return self._glsim._current_step
[docs]
def restart_from_record(self, start_rec=None):
"""
Restart a simulation from a previous record. This is useful when you
want to continue a simulation that has been interrupted.
When restarting from a record, it means that the record in question will
be deleted and simulation will go on from there.
:param start_rec: the index of the record you want to start from. If
None, the simulation will restart from the last record.
"""
# start_rec == None: restart from end of the simulation.
if not self.record_file().exists():
raise Exception("While trying to restart from previous records, I didn't "
f"find {self.record_file()}. Maybe you are trying to "
"restart a simulation that has never been started before ?")
# Load the result store with current data
self._results_store = ResultsStore( self.record_file(), mode="a")
if start_rec is not None:
self._results_store.truncate_at_report(start_rec)
t,dt,n_iter_dry_up_euler, n_iter_dry_up_rk, h,qx,qy = self._results_store.get_last_result()
# Reload the state on the step we want to start from
self.last_recorded_time = self._results_store.get_last_named_result(ResultType.T)
# We can't know the last Δt for sure because most often we record every
# N steps. Therefore the last Δt doesn't make much sense.
self.last_step_duration = None
# The first time step numbe is 0. It represents the initial conditions.
# When one asks a 100 step simulation, then we'll have the step numbers
# 0,1,...,100: 0 (I.c.) + 100 steps == 101 records.
s = self._results_store.get_last_named_result(ResultType.STEP_NUM)
self._full_timer = time.time() - self._results_store.get_last_named_result(ResultType.CLOCK_T)
self._glsim.reset_globals(self.last_recorded_time, self._glsim.time_step, s, h, qx, qy)
# Yeah, tricky. That's because what we have is the start time of the
# last record (which may be a group of simulation steps, of which we
# can't know the overall duration if it is an optimized time step) so we
# have to put ourselves back there...
self._results_store.truncate_at_report(self._results_store.nb_results - 1)
self.simulation_finished = False
logging.warning(f"Restarting from step {self._glsim._current_step}.")
[docs]
def results_path(self) -> Path:
assert self._results_store is not None, "Simulation has not produced any results"
return self._results_store.path
[docs]
def results_store(self) -> ResultsStore:
""" Gives a result store back. The result store will be opened in read mode.
"""
return ResultsStore(self.results_path(), "r")
[docs]
def record_file(self):
# Path of the result store (a directory)
return self.record_path
[docs]
def do_record(self, texture_ndx, force=False):
assert self.step_num >= 0, "must be zero based"
should_we_record = False
if force: # We can force the reporting (used in debug)
should_we_record = True
elif self._record: # Has the user enabled the reporting ?
if self._glsim.report_frequency.type == SimulationDurationType.STEPS:
should_we_record = self.step_num % self._glsim.report_frequency.duration == 0
elif self._glsim.report_frequency.type == SimulationDurationType.SECONDS:
# The simulation has already reported the information we need to
# take our decision (time information must get out of the GPU, so we don't
# have them ready all the time).
if self._glsim.last_gpu_time is not None and self.last_recorded_time is not None:
period = self._glsim.report_frequency.duration
should_we_record = floor(self._glsim.last_gpu_time / period) > floor(self.last_recorded_time / period)
emergency_stop = self.last_step_duration is not None and (isnan(self.last_step_duration) or self.last_step_duration < 0)
if should_we_record or emergency_stop:
if emergency_stop:
logging.error(f"Emergency stop! Recording additional matrices in current directory. texure_ndx was : {texture_ndx}")
np.save("q0", self._glsim.read_quantity_result(0))
np.save("q1", self._glsim.read_quantity_result(1))
np.save("alpha0", self._glsim.read_alpha(0))
np.save("alpha1", self._glsim.read_alpha(1))
with time_it("record step"):
gs = self._glsim.read_global_state()
# print(self._glsim.read_global_state().nb_active_tiles)
# print(self._glsim.read_global_state().simulation_time)
# print(
# f"read_global_state(): {gs.simulation_time} + {gs.time_step} / status={gs.status_code}"
# )
self.last_recorded_time, self.last_step_duration, nb_dryup_iterations, nb_active_tiles_cumulated = \
gs.simulation_time, gs.time_step, gs.nb_dryup_iterations, gs.total_active_tiles
# print(f"t={self.simulation_time:.4f} acive line={active_inf_line} zones={inf_zones}")
# print(self._glsim.read_active_cells())
# print(read_texture(self._glsim._infiltration_fb, GL_COLOR_ATTACHMENT0, 64, 64, GL_R32I)[55:,55:])
#assert self.last_step_duration >= 0
#print(f"\nnb_dryup_iterations: {nb_dryup_iterations}")
self.quantities = self._glsim.read_tile_packed_quantity_result(texture_ndx)
h, hu, hv = self.quantities[:,:,0], self.quantities[:,:,1], self.quantities[:,:,2]
#np.save(Path(self.record_path, f"{self._glsim.basefilename}{self.step_num:07}.GPU_RH"), h)
# if False:
# plt.imshow(h,origin='lower')
# plt.show()
if MEM_TRACKING:
self.records.append([
self.last_recorded_time, np.max(h), np.mean(h),
np.max(hu), np.max(hv)])
# Our step num has not been incremented yet, but glsim's done already.
assert self.step_num == self._glsim._current_step, \
"The steps numbers must be in sync because they're used" \
" while restarting a sim from an unfinished one " \
f"{self.step_num} == {self._glsim._current_step}"
# A mostly dry mesh is a mesh where water height is not zero but
# very small (< MOSTLY_DRY_MESHES_THRESHOLD m).
nb_mostly_dry_meshes = np.count_nonzero((h < MOSTLY_DRY_MESHES_THRESHOLD) & (h > 0))
self._results_store.append_result(
self.step_num,
self.last_recorded_time, self.last_step_duration,
nb_dryup_iterations, nb_active_tiles_cumulated,
h, hu, hv, None, None, clock_time=time.time() - self._full_timer,
delta_early_out=self._last_early_out_delta or 0,
nb_mostly_dry_meshes=nb_mostly_dry_meshes)
if self._enable_alpha_recording:
# Stores the last alpha maps
# FIXME I save the two of them because the correct one to take
# depends on the number of dry up cancellation iterations that were
# done and I don't know that yet (I should put it in the
# globals).
alpha_correction = self._glsim.read_tile_packed_alpha(0)
self._results_store.append_additional_result("alpha_values0", alpha_correction[:,:,0])
self._results_store.append_additional_result("alpha_masks0", alpha_correction[:,:,1])
alpha_correction = self._glsim.read_tile_packed_alpha(1)
self._results_store.append_additional_result("alpha_values1", alpha_correction[:,:,0])
self._results_store.append_additional_result("alpha_masks1", alpha_correction[:,:,1])
active_cells_packed = self._glsim.read_active_cells_map()
self._results_store.append_additional_result(
"active_cells_packed", active_cells_packed
)
# alpha_correction = read_texture(
# self._glsim.access_fb, GL_COLOR_ATTACHMENT0, self._glsim.width, self._glsim.height, GL_RGB32F)
# self._results_store.additional_result("alpha", alpha_correction)
# if self._glsim._debugging:
# dbg1 = read_texture(self._glsim.access_fb,
# GL_COLOR_ATTACHMENT2, self._glsim.width, self._glsim.height, GL_RGBA32F)
# self._results_store.additional_result("debug1", dbg1)
if emergency_stop:
msg = f"NaN's were detected, simulation has been saved. Last written flip/flop={texture_ndx}"
logging.error(msg)
self._results_store.close()
raise Exception(msg)
return (h, hu, hv)
else:
return None
[docs]
def texture_written_by_last_step(self) -> int:
""" After running `run_one_step` this gives the number of the texture
that was *written* to (zero or one; rememeber the GPU code often uses
flip-flop textures, that's what you query here).
"""
return self.current_quantity
[docs]
def texture_read_by_last_step(self) -> int:
return 1 - self.current_quantity
[docs]
def run_one_step(self,dont_close_results=False):
if self.simulation_finished:
raise Exception("Simulation is finished, you can't run it anymore")
# step_num is currently self._glsim._current_step
if self.step_num == 0:
# Recording initial conditions first
recorded_res = self.do_record(texture_ndx=0, force=True)
with time_it("Run one step"):
source_quantity = self.current_quantity
dest_quantity = 1 - source_quantity
self._glsim.simulation_step(source_quantity)
# self._glsim._current_step is increased at the end of a simulation step
recorded_res = self.do_record(dest_quantity)
# FIXME REAAALLLLY dirty. I do that because I need glsim to generate the "generated_constans.frs" include
# before I can load these shaders...
if not hasattr(self, "_vertex_shader2"):
self._vertex_shader2 = load_shader_from_file(Path(SHADER_PATH,"simplevertexshader.vs"))
self._active_tiles_shader2 = load_shader_from_file(Path(SHADER_PATH,"active_tiles_quad.gs"))
self._texture_viewer_shader2 = load_shader_from_file(Path(SHADER_PATH,"textureViewerShader.frs"))
self._texture_viewer_program2 = load_program(self._vertex_shader2, self._texture_viewer_shader2, self._active_tiles_shader2)
self._vertex_shader = load_shader_from_file(Path(SHADER_PATH,"simplevertexshader.vs"))
self._active_tiles_shader = load_shader_from_file(Path(SHADER_PATH,"active_tiles_quad.gs"))
self._texture_viewer_shader = load_shader_from_file(Path(SHADER_PATH,"textureViewerShader.frs"))
self._texture_viewer_program = load_program(self._vertex_shader, self._texture_viewer_shader)
with time_it("Run one step - admin"):
if (self._glsim.simulation_duration.type == SimulationDurationType.STEPS and self.step_num >= self._glsim.simulation_duration.duration
or self._glsim.simulation_duration.type == SimulationDurationType.SECONDS and self.last_recorded_time > self._glsim.simulation_duration.duration):
#print(f"Finished {self.step_num} >= {self._glsim.simulation_duration}")
self.simulation_finished = True
# --------------------------------------------------------------------
# Flip/flop the (source) current_quantity index
if self.current_quantity == 0:
self.current_quantity = 1
else:
self.current_quantity = 0
# --------------------------------------------------------------------
# Record the result of this step
if True:
# This little hac to make sure the first report is the initial condition.
# This is inline with what we do in rust.
if recorded_res is not None:
h, hu, hv = recorded_res
# We end the simulation if things don't move anymore
if self._previous_results is None:
self._previous_results = (h, hu, hv)
else:
prev_h, prev_hu, prev_hv = self._previous_results
delta_h = np.abs(h - prev_h)
delta_hu = np.abs(hu - prev_hu)
delta_hv = np.abs(hv - prev_hv)
sum_delta = np.sum(delta_h + delta_hu + delta_hv)
self._previous_results = (h, hu, hv)
self._last_early_out_delta = sum_delta
if self._early_out_delta_max is not None:
#print(sum_delta)
if sum_delta <= self._early_out_delta_max:
self._still_simulation_count += 1
if self._still_simulation_count >= 3:
logging.warning(f"Simulation is still, stopping at t={self.last_recorded_time}s after {self.step_num} iterations.")
self.simulation_finished = True
self.cancelled_early = True
else:
logging.info(f"Simulation is slowing down: |Δh|+|ΔQx|+|ΔQy| = {sum_delta}")
else:
# Reset count
self._still_simulation_count = 0
with time_it("Recording at step"):
if self.simulation_finished and not dont_close_results:
# if self.record:
# np.save(Path(self.record_path,"data"), np.array(self.records))
# #np.save(Path(self.record_path,"quantities"), self.quantities)
if self._results_store is not None:
self._results_store.close()
with time_it("Refresh zoom"):
if self._refresh_zoom_timer.has_shot():
# Recompute the min/max of the array
# min/max is used to improve the display readability
with time_it("Refresh zoom, read_tile_packed_quantity_result"):
#print("read qty")
p = self._glsim.read_tile_packed_quantity_result(0) # 1-self.current_quantity)
h = p[:,:,0] # read water height
valid_heights = h[h>0] # Avoid mean on empty arrays
if valid_heights.any():
self._mean = (np.mean(valid_heights) + self._mean)/2
std = np.std(h[h>0])
else:
self._mean = 0
std = 0
# p = self._glsim.read_tile_packed_quantity_result(self.current_quantity)
# h = p[:,:,0] # read water height
# h_max = max(np.max(h), h_max)
# How this works:
# I average the old zoom value and the new one.
# I want color to be distributed around the mean (0.5), from 0 to 1.
# So the max deviation is 0.5.
# I want to see 2 std => 2*std
# I want everythign to be xformed back into 0..1 => I divide by 2*std.
# If std happens to be zero, then we set it to 0.1
self._zoom = (0.5/(2*max(0.1,std)) + self._zoom)/2
logging.debug(f"Rezooming at mean={self._mean:3g}, zoom={self._zoom:3g}")
# if self.h_convergence and self._last_h is not None:
# if abs(np.max(h - self._last_h)) < self.h_convergence:
# logging.info("Simulation has converged")
# self.simulation_finished = True
# self._last_h = h
[docs]
def plot_progress2(self):
# FIXME Accessing _glsim's private stuff is bad abstraction...
domain_size = self._glsim.width, self._glsim.height
uniforms = {
"zoom": float(self._zoom),
"mean": float(self._mean),
# FragCoord (screen) to texture coord
"textureCoordinateTransform": (
float(domain_size[0]/self._gl_window_viewport[2]),
float(domain_size[1]/self._gl_window_viewport[3]))
}
textures= {
"quantityTexture": self._glsim.quantity_tex[1-self.current_quantity],
"tileIndirection" : self._glsim._tile_indirection_tex,
}
#print(self._glsim.nbx, self._glsim.nby)
glUseProgram(self._texture_viewer_program2)
glBindFramebuffer(GL_FRAMEBUFFER, GL_NONE) # draw on screen
glClear(GL_COLOR_BUFFER_BIT)
self._glsim._draw_active_tiles(self._texture_viewer_program2, uniforms, textures, self._gl_window_viewport)
# def plot_progress(self):
# if self._glsim._optim_geom_shaders:
# return self.plot_progress2()
# # glUseProgram(self._texture_viewer_program)
# # #self._zoom = 0.5
# # set_uniform(self._texture_viewer_program, "zoom", float(self._zoom))
# # set_uniform(self._texture_viewer_program, "mean", float(self._mean))
# # # For reaon unbeknown to me, this doesn't give what I want
# # # as window width/height...
# # # window_width, window_height = pygame.display.get_surface().get_size()
# # r = (self._glsim.nbx/self._glsim._window_width,
# # self._glsim.nby/self._glsim._window_height)
# # set_uniform(self._texture_viewer_program, "textureCoordinateTransform", r)
# # glBindFramebuffer(GL_FRAMEBUFFER, GL_NONE) # draw on screen
# # glClear(GL_COLOR_BUFFER_BIT)
# # # FIXME Encapsulation is no good here
# # set_texture(self._texture_viewer_program, "sTexture",
# # self._glsim.quantity_tex[1-self.current_quantity], 0)
# # x,y,w,h = 0,0,self._glsim._window_width,self._glsim._window_height
# # self._glsim._draw_active_tiles(self._texture_viewer_program,x,y,w,h)
# # # self._glsim._draw_active_tiles(self._texture_viewer_program,
# # # 0, self._glsim.height//4,
# # # self._glsim.width, self._glsim.height//4)
# def has_handle(self, fpath):
# for proc in psutil.process_iter():
# try:
# for item in proc.open_files():
# if fpath == item.path:
# return True
# except Exception as ex:
# print(ex)
# pass
# return False
[docs]
def file_in_use(self, f):
try:
os.rename(f, f)
return False
except OSError:
return True
[docs]
def _try_reload_bathymetry(self):
try:
# Using a try catch seems overkill, but we had something
# like that happening in 2023. Error was:
# [WinError 5] Accès refusé ...
bath_exists = self._bathy_file.exists()
except Exception as ex:
logging.error(f"Can't check if {self._bathy_file} exists: {ex}")
bath_exists = False
if bath_exists:
try:
new_time = os.stat(str(self._bathy_file)).st_mtime
if new_time != self._bathy_file_mtime:
print() # Nicer CLI
logging.info(f"Loading bathymetry file for update: {self._bathy_file}")
self._bathy_file_mtime = new_time
if not self.file_in_use(str(self._bathy_file)):
# from wolfhece.wolf_array import WolfArray_Sim2D
# a = WolfArray_Sim2D(fname=str(self._bathy_file))
# a.preload = True
# a.read_all()
# logging.info(f"Bathymetry file shape is {a.array.shape} {a.nbx}x{a.nby}")
# self._glsim.set_buffers(bathymetry_array=np.ascontiguousarray(np.transpose(a.array)))
try:
# We transpose because we expect the user will pass us an array
# following the usual Wolf convention.
a = np.load(self._bathy_file).transpose()
except Exception as ex:
logging.error(f"Can't load bathymetry file {self._bathy_file}. Numpy can't read it, it says: \"{ex}\"")
return
if a.shape != (self._glsim.nby, self._glsim.nbx):
if self._glsim.nby == a.shape[1] and self._glsim.nbx == a.shape[0]:
suggestion = " Maybe you need to transpose ?"
else:
suggestion = ""
logging.error(f"The shape of the bathymetry update file is not correct. You gave x={a.shape[0]}, y={a.shape[1]} columns. I expect {self._glsim.nbx} and {self._glsim.nby}.{suggestion}")
return
self._glsim.set_buffers(bathymetry_array=a)
logging.info(f"Loaded bathymetry file {self._bathy_file}, shape is {a.shape}")
if self._results_store is not None:
gs = self._glsim.read_global_state()
self._results_store.add_event(SimulationEventType.BathymetryUpdate, simulation_time= gs.simulation_time, simulation_step=self.step_num, data=self._bathy_file)
# z = np.zeros((128,128), dtype=np.float32)
# z[:,0:10] = 100
# self._glsim.set_buffers(bathymetry_array=z)
else:
logging.error(f"The bathymetry file {self.record_path} is opened by another process")
# else:
# logging.warn(f"The bathymetry file {self.record_path} didn't change since the last time I've seen it")
except Exception as ex:
logging.error(f"Something went wrong: {ex}")
[docs]
def init_result_store(self):
if self._record:
if self._results_store is None:
self._results_store = ResultsStore( self.record_file(), mode="w", tile_packer=self._glsim.tile_packer())
[docs]
def full_run(self, pace=None, refresh_view=1, label="sim.pdf", dont_close_results=False):
logging.debug(f"FINISH={FINISH} STRIPES={STRIPES_ON_NAP}")
if self._glsim.nb_active_meshes <= 64*64:
# If sim too small, then one-shot textures get in the way.
warning = "(!!! very approximate !!!)"
else:
warning = "(approximate)"
logging.info(f"Total memory for textures: {total_textures_size()/1000_000:.1f}Mb, bytes per mesh {warning}:{int(total_textures_size()/self._glsim.nb_active_meshes)}")
# Make sure the reporting frequency is meaningful
if self._glsim.report_frequency.type == self._glsim.simulation_duration.type:
assert self._glsim.report_frequency.duration <= self._glsim.simulation_duration.duration, \
f"Report frequency {self._glsim.report_frequency} is above simulation's duration {self._glsim.simulation_duration}"
# debug
if self._full_timer is None:
# Init unless already initialized (most likely because we restarted a simulation)
self._full_timer = time.time()
complete_page_flip = True
if MEM_TRACKING:
mem_tracker = tracker.SummaryTracker()
# {percentage:3.0f}%
with tqdm.tqdm(
total=100, bar_format="{desc}|{bar}|{postfix}", ncols=80
) as taqadum:
running = True
# Inidcates if the user or the early-out condition was met before
# the planned end of the simulation.
self.cancelled_early = False
refresh_view_timer = EveryNSeconds(refresh_view)
refresh_logger_timer = EveryNSeconds(0.5)
logging_iterations = 0
NB_ITER_STATS=100
iteration_times = np.zeros( (NB_ITER_STATS,))
# result store might be already opened if the user wants
# to continue an existing simulation.
self.init_result_store()
while running and not self.simulation_finished:
self.run_one_step(dont_close_results)
#tic: float = time.perf_counter()
with time_it("-- Page flip"):
# FIXME don't forget to remove the True !
refresh = refresh_view_timer.has_shot()
if (self.step_num == 1 or refresh or complete_page_flip):
#print(f"progress {refresh} {complete_page_flip} {self.step_num}")
# Yeah, two plot progress else double buffering
# shows flicker
self.plot_progress2()
if refresh:
complete_page_flip = True
else:
complete_page_flip = False
# print("yo-1b")
# print("yo-1c")
# self.plot_progress()
# print("yo-1d")
# FIXME This must be done on each frame, else the GPU
# doesn't compute anything... Why ? Do I understand this issue ?
self._gl_page_flip_func()
if True:
if MEM_TRACKING and self.step_num % 40 == 0:
mem_tracker.print_diff()
log_mem_used()
with time_it("-- admin tasks"):
iteration_times[self.step_num % NB_ITER_STATS] = time.time()
if self.step_num > NB_ITER_STATS:
# step_num+1 MOD ... is the *first* in the list !!!
stats_total_duration = iteration_times[self.step_num % NB_ITER_STATS] - iteration_times[(self.step_num +1) % NB_ITER_STATS]
average_duration = stats_total_duration / NB_ITER_STATS
active_meshes_per_second = self._glsim.nb_active_meshes/average_duration
pt = f"{(active_meshes_per_second/1_000_000)/average_duration:.1f}"
self.meshes_per_second = pt
else:
pt = "?? "
#print("yo-4")
# Round so that we actually see the time step increments
#print(f"debug: {self._glsim.last_step_duration}")
# self.last_step_duration > 0, the step duration is recorded at the beginning
# of the iteration => it may have never been initialized (at begin of sim, or at beginiing of restart)
if self._glsim.current_simulation_time_step is not None and self._glsim.current_simulation_time_step > 0:
# total_time_simulation_str = str(
# round(self._glsim.last_gpu_time,
# ceil(abs(log10(self._glsim.current_simulation_time_step))))) + "s"
total_time_simulation_str = seconds_to_duration_str(self._glsim.last_gpu_time)
last_duration_str = f"{self._glsim.current_simulation_time_step:.4g}s"
elif self._glsim.last_gpu_time is not None:
total_time_simulation_str = seconds_to_duration_str(self._glsim.last_gpu_time)
last_duration_str = "--"
else:
total_time_simulation_str = "--"
last_duration_str = "--"
if self._glsim.simulation_duration.type == SimulationDurationType.SECONDS:
#print(100 * self.last_recorded_time / self._glsim.simulation_duration.duration)
percent_done = 100* max(self.last_recorded_time or 0, self._glsim.last_gpu_time or 0) / self._glsim.simulation_duration.duration
else:
percent_done = 100*self.step_num / self._glsim.simulation_duration.duration
percent_done = min(round(percent_done),100)
stats_total_duration = iteration_times[self.step_num % NB_ITER_STATS] - iteration_times[(self.step_num +1) % NB_ITER_STATS]
average_duration = stats_total_duration / NB_ITER_STATS
iter_per_sec = 1 / average_duration
logging_iterations += 1
if refresh_logger_timer.has_shot() or self.simulation_finished:
if self._record:
if self._results_store.nb_results > 1:
rec_flag = f"[{self._results_store.nb_results} records] "
elif self._results_store.nb_results == 1:
rec_flag = f"[{self._results_store.nb_results} record] "
else:
rec_flag = "[No record]"
else:
rec_flag = ""
taqadum.set_postfix_str(f"{iter_per_sec:.2f} it./sec.")
taqadum.set_description(
f"{rec_flag} t={total_time_simulation_str} Δt={last_duration_str}"
)
# FIXME Reenable this but with correct values :-)
# , {pt}M meshes/s ({self._glsim.nb_active_meshes/1_000_000:.2f} active) {self.last_delta or 0:g}
taqadum.update(percent_done - taqadum.n)
logging_iterations = 0
self._try_reload_bathymetry()
if pace:
raise Exception("No pace during testing")
time.sleep(pace)
# FIXME Dirty. We're bypassing a glfw context here...
if self._gl_page_flip_func == pygame.display.flip:
for event in pygame.event.get():
if event.type == pygame.QUIT:
running = False
self.cancelled_early = True
elif event.type == pygame.KEYDOWN:
if event.key in (pygame.K_ESCAPE, pygame.K_q):
running = False
self.cancelled_early = True
#print(time.perf_counter() - tic)
if self.cancelled_early:
# When you stop early or by hand int he middle of two record steps
# which are 1 hours apart, you don't want to loose too much
# information (say half an hour). So we record the last moment.
self.do_record(texture_ndx=1 - self.current_quantity, force=True)
# Yes, I had the equality once. So, you see, these things, they happen.
new_time = time.time()
while new_time == self._full_timer:
new_time = time.time()
self.total_duration = time.time() - self._full_timer
self.iter_per_second = self.step_num / self.total_duration
if self._record:
logging.info(f"WolfGPU records stored in {self.record_file().resolve()}")
logging.debug(f"Last delta = {self._last_early_out_delta}.")
if self.step_num >= NB_ITER_STATS:
stats_total_duration = iteration_times[self.step_num % NB_ITER_STATS] - iteration_times[(self.step_num +1) % NB_ITER_STATS]
average_duration = stats_total_duration / NB_ITER_STATS
if self._record:
recs = f"{self._results_store.nb_results} records."
else:
recs = "No records."
logging.info(f"{self.iter_per_second:g} iter/sec overall. {1/average_duration:.2f} it./s over the last {NB_ITER_STATS} iterations. {self.step_num} steps. {recs}")