diff --git a/mcdc/input_.py b/mcdc/input_.py index e54124e3..245ea5c1 100644 --- a/mcdc/input_.py +++ b/mcdc/input_.py @@ -1401,7 +1401,7 @@ def domain_decomposition( x=None, y=None, z=None, - exchange_rate=100, + exchange_rate=100000, work_ratio=None, repro=True, ): @@ -1417,9 +1417,9 @@ def domain_decomposition( z : array_like[float], optional Location of subdomain boundaries in z (default None). exchange_rate : float, optional - number of particles to acumulate in the domain banks before sending. + Number of particles to acumulate in the domain banks before sending. work_ratio : array_like[integer], optional - Inte + Number of processors in each domain Returns ------- @@ -1428,7 +1428,6 @@ def domain_decomposition( """ card = mcdc.input_deck.technique card["domain_decomposition"] = True - card["domain_bank_size"] = int(1e5) card["dd_exchange_rate"] = int(exchange_rate) card["dd_repro"] = repro dom_num = 1 diff --git a/mcdc/kernel.py b/mcdc/kernel.py index f5fb68bb..aee32a29 100644 --- a/mcdc/kernel.py +++ b/mcdc/kernel.py @@ -42,28 +42,28 @@ def domain_crossing(P, mcdc): flag = directions[0] # Score on tally if flag == MESH_X and P["ux"] > 0: - add_particle(copy_particle(P), mcdc["bank_domain_xp"]) - if mcdc["bank_domain_xp"]["size"] == max_size: + add_particle(copy_particle(P), mcdc["domain_decomp"]["bank_xp"]) + if mcdc["domain_decomp"]["bank_xp"]["size"] == max_size: dd_particle_send(mcdc) if flag == MESH_X and P["ux"] < 0: - add_particle(copy_particle(P), mcdc["bank_domain_xn"]) - if mcdc["bank_domain_xn"]["size"] == max_size: + add_particle(copy_particle(P), mcdc["domain_decomp"]["bank_xn"]) + if mcdc["domain_decomp"]["bank_xn"]["size"] == max_size: dd_particle_send(mcdc) if flag == MESH_Y and P["uy"] > 0: - add_particle(copy_particle(P), mcdc["bank_domain_yp"]) - if mcdc["bank_domain_yp"]["size"] == max_size: + add_particle(copy_particle(P), mcdc["domain_decomp"]["bank_yp"]) + if mcdc["domain_decomp"]["bank_yp"]["size"] == max_size: dd_particle_send(mcdc) if flag == MESH_Y and P["uy"] < 0: - add_particle(copy_particle(P), mcdc["bank_domain_yn"]) - if mcdc["bank_domain_yn"]["size"] == max_size: + add_particle(copy_particle(P), mcdc["domain_decomp"]["bank_yn"]) + if mcdc["domain_decomp"]["bank_yn"]["size"] == max_size: dd_particle_send(mcdc) if flag == MESH_Z and P["uz"] > 0: - add_particle(copy_particle(P), mcdc["bank_domain_zp"]) - if mcdc["bank_domain_zp"]["size"] == max_size: + add_particle(copy_particle(P), mcdc["domain_decomp"]["bank_zp"]) + if mcdc["domain_decomp"]["bank_zp"]["size"] == max_size: dd_particle_send(mcdc) if flag == MESH_Z and P["uz"] < 0: - add_particle(copy_particle(P), mcdc["bank_domain_zn"]) - if mcdc["bank_domain_zn"]["size"] == max_size: + add_particle(copy_particle(P), mcdc["domain_decomp"]["bank_zn"]) + if mcdc["domain_decomp"]["bank_zn"]["size"] == max_size: dd_particle_send(mcdc) P["alive"] = False @@ -73,196 +73,310 @@ def domain_crossing(P, mcdc): # ============================================================================= +requests = [] + + +def save_request(req_pair): + global requests + + updated_requests = [] + + status = MPI.Status() + for req, buf in requests: + if not req.Test(status): + updated_requests.append((req, buf)) + + updated_requests.append(req_pair) + requests = updated_requests + + +def clear_requests(): + global requests + for req, buf in requests: + req.Free() + + requests = [] + + @njit -def dd_particle_send(mcdc): +def dd_check_halt(mcdc): + return mcdc["domain_decomp"]["work_done"] + + +@njit +def dd_check_in(mcdc): + mcdc["domain_decomp"]["send_count"] = 0 + mcdc["domain_decomp"]["recv_count"] = 0 + mcdc["domain_decomp"]["send_total"] = 0 + mcdc["domain_decomp"]["rank_busy"] = True + + with objmode(rank="int64", total="int64"): + rank = MPI.COMM_WORLD.Get_rank() + total = MPI.COMM_WORLD.Get_size() + + if rank == 0: + mcdc["domain_decomp"]["busy_total"] = total + else: + mcdc["domain_decomp"]["busy_total"] = 0 + + +@njit +def dd_check_out(mcdc): with objmode(): - for i in range( - max( - len(mcdc["technique"]["dd_xp_neigh"]), - len(mcdc["technique"]["dd_xn_neigh"]), - len(mcdc["technique"]["dd_yp_neigh"]), - len(mcdc["technique"]["dd_yn_neigh"]), - len(mcdc["technique"]["dd_zp_neigh"]), - len(mcdc["technique"]["dd_zn_neigh"]), + rank = MPI.COMM_WORLD.Get_rank() + send_count = mcdc["domain_decomp"]["send_count"] + recv_count = mcdc["domain_decomp"]["recv_count"] + send_total = mcdc["domain_decomp"]["send_total"] + busy_total = mcdc["domain_decomp"]["busy_total"] + rank_busy = mcdc["domain_decomp"]["rank_busy"] + + if send_count != 0: + print( + f"Domain decomposed loop closed out with non-zero send count {send_count} in rank {rank}" ) - ): - if mcdc["technique"]["dd_xp_neigh"].size > i: - size = mcdc["bank_domain_xp"]["size"] - ratio = int(size / len(mcdc["technique"]["dd_xp_neigh"])) - start = ratio * i - end = start + ratio - if i == len(mcdc["technique"]["dd_xp_neigh"]) - 1: - end = size - bank = np.array(mcdc["bank_domain_xp"]["particles"][start:end]) - request1 = MPI.COMM_WORLD.send( - bank, dest=mcdc["technique"]["dd_xp_neigh"][i], tag=1 - ) + mcdc["domain_decomp"]["send_count"] = 0 - if mcdc["technique"]["dd_xn_neigh"].size > i: - size = mcdc["bank_domain_xn"]["size"] - ratio = int(size / len(mcdc["technique"]["dd_xn_neigh"])) - start = ratio * i - end = start + ratio - if i == len(mcdc["technique"]["dd_xn_neigh"]) - 1: - end = size - bank = np.array(mcdc["bank_domain_xn"]["particles"][start:end]) - request2 = MPI.COMM_WORLD.send( - bank, dest=mcdc["technique"]["dd_xn_neigh"][i], tag=2 - ) + if recv_count != 0: + print( + f"Domain decomposed loop closed out with non-zero recv count {recv_count} in rank {rank}" + ) + mcdc["domain_decomp"]["recv_count"] = 0 - if mcdc["technique"]["dd_yp_neigh"].size > i: - size = mcdc["bank_domain_yp"]["size"] - ratio = int(size / len(mcdc["technique"]["dd_yp_neigh"])) - start = ratio * i - end = start + ratio - if i == len(mcdc["technique"]["dd_yp_neigh"]) - 1: - end = size - bank = np.array(mcdc["bank_domain_yp"]["particles"][start:end]) - request3 = MPI.COMM_WORLD.send( - bank, dest=mcdc["technique"]["dd_yp_neigh"][i], tag=3 - ) + if send_total != 0: + print( + f"Domain decomposed loop closed out with non-zero send total {send_total} in rank {rank}" + ) + mcdc["domain_decomp"]["send_total"] = 0 - if mcdc["technique"]["dd_yn_neigh"].size > i: - size = mcdc["bank_domain_yn"]["size"] - ratio = int(size / len(mcdc["technique"]["dd_yn_neigh"])) - start = ratio * i - end = start + ratio - if i == len(mcdc["technique"]["dd_yn_neigh"]) - 1: - end = size - bank = np.array(mcdc["bank_domain_yn"]["particles"][start:end]) - request4 = MPI.COMM_WORLD.send( - bank, dest=mcdc["technique"]["dd_yn_neigh"][i], tag=4 - ) + if busy_total != 0: + print( + f"Domain decomposed loop closed out with non-zero busy total {busy_total} in rank {rank}" + ) + mcdc["domain_decomp"]["busy_total"] = 0 - if mcdc["technique"]["dd_zp_neigh"].size > i: - size = mcdc["bank_domain_zp"]["size"] - ratio = int(size / len(mcdc["technique"]["dd_zp_neigh"])) - start = ratio * i - end = start + ratio - if i == len(mcdc["technique"]["dd_zp_neigh"]) - 1: - end = size - bank = np.array(mcdc["bank_domain_zp"]["particles"][start:end]) - request5 = MPI.COMM_WORLD.send( - bank, dest=mcdc["technique"]["dd_zp_neigh"][i], tag=5 - ) + if rank_busy: + print( + f"Domain decomposed loop closed out with rank {rank} still marked as busy" + ) + mcdc["domain_decomp"]["rank_busy"] = 0 - if mcdc["technique"]["dd_zn_neigh"].size > i: - size = mcdc["bank_domain_zn"]["size"] - ratio = int(size / len(mcdc["technique"]["dd_zn_neigh"])) - start = ratio * i - end = start + ratio - if i == len(mcdc["technique"]["dd_zn_neigh"]) - 1: - end = size - bank = np.array(mcdc["bank_domain_zn"]["particles"][start:end]) - request6 = MPI.COMM_WORLD.send( - bank, dest=mcdc["technique"]["dd_zn_neigh"][i], tag=6 - ) + clear_requests() - sent_particles = ( - mcdc["bank_domain_xp"]["size"] - + mcdc["bank_domain_xn"]["size"] - + mcdc["bank_domain_yp"]["size"] - + mcdc["bank_domain_yn"]["size"] - + mcdc["bank_domain_zp"]["size"] - + mcdc["bank_domain_zn"]["size"] - ) - mcdc["technique"]["dd_sent"] += sent_particles - mcdc["bank_domain_xp"]["size"] = 0 - mcdc["bank_domain_xn"]["size"] = 0 - mcdc["bank_domain_yp"]["size"] = 0 - mcdc["bank_domain_yn"]["size"] = 0 - mcdc["bank_domain_zp"]["size"] = 0 - mcdc["bank_domain_zn"]["size"] = 0 +@njit +def dd_signal_halt(mcdc): + with objmode(): + for rank in range(1, MPI.COMM_WORLD.Get_size()): + dummy_buff = np.zeros((1,), dtype=np.int32) + MPI.COMM_WORLD.Send(dummy_buff, dest=rank, tag=3) -# ============================================================================= -# Recieve particles and clear banks -# ============================================================================= + mcdc["domain_decomp"]["work_done"] = True @njit -def dd_particle_receive(mcdc): - buff = np.zeros( - mcdc["bank_domain_xp"]["particles"].shape[0], dtype=type_.particle_record - ) +def dd_signal_block(mcdc): - with objmode(size="int64"): - bankr = mcdc["bank_active"]["particles"][:0] - size_old = bankr.shape[0] - for i in range( - max( - len(mcdc["technique"]["dd_xp_neigh"]), - len(mcdc["technique"]["dd_xn_neigh"]), - len(mcdc["technique"]["dd_yp_neigh"]), - len(mcdc["technique"]["dd_yn_neigh"]), - len(mcdc["technique"]["dd_zp_neigh"]), - len(mcdc["technique"]["dd_zn_neigh"]), + with objmode(rank="int64"): + rank = MPI.COMM_WORLD.Get_rank() + + send_delta = ( + mcdc["domain_decomp"]["send_count"] - mcdc["domain_decomp"]["recv_count"] + ) + if rank == 0: + mcdc["domain_decomp"]["send_total"] += send_delta + mcdc["domain_decomp"]["busy_total"] -= 1 + else: + with objmode(): + buff = np.zeros((1,), dtype=type_.dd_turnstile_event) + buff[0]["busy_delta"] = -1 + buff[0]["send_delta"] = send_delta + req = MPI.COMM_WORLD.Isend( + [buff, type_.dd_turnstile_event_mpi], dest=0, tag=2 ) + save_request((req, buff)) + + mcdc["domain_decomp"]["send_count"] = 0 + mcdc["domain_decomp"]["recv_count"] = 0 + + if ( + (rank == 0) + and (mcdc["domain_decomp"]["busy_total"] == 0) + and (mcdc["domain_decomp"]["send_total"] == 0) + ): + dd_signal_halt(mcdc) + + +@njit +def dd_signal_unblock(mcdc): + + with objmode(rank="int64"): + rank = MPI.COMM_WORLD.Get_rank() + + send_delta = ( + mcdc["domain_decomp"]["send_count"] - mcdc["domain_decomp"]["recv_count"] + ) + + if rank == 0: + mcdc["domain_decomp"]["send_total"] += send_delta + mcdc["domain_decomp"]["busy_total"] += 1 + if (mcdc["domain_decomp"]["busy_total"] == 0) and ( + mcdc["domain_decomp"]["send_total"] == 0 ): - if mcdc["technique"]["dd_xp_neigh"].size > i: - received1 = MPI.COMM_WORLD.irecv( - source=mcdc["technique"]["dd_xp_neigh"][i], tag=2 + dd_signal_halt(mcdc) + else: + with objmode(): + buff = np.zeros((1,), dtype=type_.dd_turnstile_event) + buff[0]["busy_delta"] = 1 + buff[0]["send_delta"] = send_delta + req = MPI.COMM_WORLD.Isend( + [buff, type_.dd_turnstile_event_mpi], dest=0, tag=2 + ) + save_request((req, buff)) + mcdc["domain_decomp"]["send_count"] = 0 + mcdc["domain_decomp"]["recv_count"] = 0 + + +@njit +def dd_distribute_bank(mcdc, bank, dest_list): + + with objmode(send_delta="int64"): + dest_count = len(dest_list) + send_delta = 0 + for i, dest in enumerate(dest_list): + size = bank["size"] + ratio = int(size / dest_count) + start = ratio * i + end = start + ratio + if i == dest_count - 1: + end = size + sub_bank = np.array(bank["particles"][start:end]) + if sub_bank.shape[0] > 0: + req = MPI.COMM_WORLD.Isend( + [sub_bank, type_.particle_record_mpi], dest=dest, tag=1 ) - if received1.Get_status(): - bankr = np.append(bankr, received1.wait()) - else: - MPI.Request.cancel(received1) + save_request((req, sub_bank)) + send_delta += end - start - if mcdc["technique"]["dd_xn_neigh"].size > i: - received2 = MPI.COMM_WORLD.irecv( - source=mcdc["technique"]["dd_xn_neigh"][i], tag=1 - ) - if received2.Get_status(): - bankr = np.append(bankr, received2.wait()) - else: - MPI.Request.cancel(received2) + mcdc["domain_decomp"]["send_count"] += send_delta + bank["size"] = 0 - if mcdc["technique"]["dd_yp_neigh"].size > i: - received3 = MPI.COMM_WORLD.irecv( - source=mcdc["technique"]["dd_yp_neigh"][i], tag=4 - ) - if received3.Get_status(): - bankr = np.append(bankr, received3.wait()) - else: - MPI.Request.cancel(received3) - if mcdc["technique"]["dd_yn_neigh"].size > i: - received4 = MPI.COMM_WORLD.irecv( - source=mcdc["technique"]["dd_yn_neigh"][i], tag=3 - ) - if received4.Get_status(): - bankr = np.append(bankr, received4.wait()) - else: - MPI.Request.cancel(received4) +@njit +def dd_particle_send(mcdc): + dd_distribute_bank( + mcdc, mcdc["domain_decomp"]["bank_xp"], mcdc["technique"]["dd_xp_neigh"] + ) + dd_distribute_bank( + mcdc, mcdc["domain_decomp"]["bank_xn"], mcdc["technique"]["dd_xn_neigh"] + ) + dd_distribute_bank( + mcdc, mcdc["domain_decomp"]["bank_yp"], mcdc["technique"]["dd_yp_neigh"] + ) + dd_distribute_bank( + mcdc, mcdc["domain_decomp"]["bank_yn"], mcdc["technique"]["dd_yn_neigh"] + ) + dd_distribute_bank( + mcdc, mcdc["domain_decomp"]["bank_zp"], mcdc["technique"]["dd_zp_neigh"] + ) + dd_distribute_bank( + mcdc, mcdc["domain_decomp"]["bank_zn"], mcdc["technique"]["dd_zn_neigh"] + ) - if mcdc["technique"]["dd_zp_neigh"].size > i: - received5 = MPI.COMM_WORLD.irecv( - source=mcdc["technique"]["dd_zp_neigh"][i], tag=6 - ) - if received5.Get_status(): - bankr = np.append(bankr, received5.wait()) - else: - MPI.Request.cancel(received5) - if mcdc["technique"]["dd_zn_neigh"].size > i: - received6 = MPI.COMM_WORLD.irecv( - source=mcdc["technique"]["dd_zn_neigh"][i], tag=5 - ) - if received6.Get_status(): - bankr = np.append(bankr, received6.wait()) - else: - MPI.Request.cancel(received6) +# ============================================================================= +# Receive particles and clear banks +# ============================================================================= - size = bankr.shape[0] - # Set output buffer - for i in range(size): - buff[i] = bankr[i] + +@njit +def dd_get_recv_tag(): + + with objmode(tag="int64"): + status = MPI.Status() + MPI.COMM_WORLD.Probe(status=status) + tag = status.Get_tag() + + return tag + + +@njit +def dd_recv_particles(mcdc): + + buff = np.zeros( + mcdc["domain_decomp"]["bank_zp"]["particles"].shape[0], + dtype=type_.particle_record, + ) + + with objmode(size="int64"): + status = MPI.Status() + MPI.COMM_WORLD.Recv([buff, type_.particle_record_mpi], status=status) + size = status.Get_count(type_.particle_record_mpi) + rank = MPI.COMM_WORLD.Get_rank() + + mcdc["domain_decomp"]["recv_count"] += size # Set source bank from buffer for i in range(size): add_particle(buff[i], mcdc["bank_active"]) - mcdc["technique"]["dd_sent"] -= size + + if ( + mcdc["domain_decomp"]["recv_count"] > 0 + and not mcdc["domain_decomp"]["rank_busy"] + ): + dd_signal_unblock(mcdc) + mcdc["domain_decomp"]["rank_busy"] = True + + +@njit +def dd_recv_turnstile(mcdc): + + with objmode(busy_delta="int64", send_delta="int64"): + event_buff = np.zeros((1,), dtype=type_.dd_turnstile_event) + MPI.COMM_WORLD.Recv([event_buff, type_.dd_turnstile_event_mpi]) + busy_delta = event_buff[0]["busy_delta"] + send_delta = event_buff[0]["send_delta"] + rank = MPI.COMM_WORLD.Get_rank() + busy_total = mcdc["domain_decomp"]["busy_total"] + send_total = mcdc["domain_decomp"]["send_total"] + + mcdc["domain_decomp"]["busy_total"] += busy_delta + mcdc["domain_decomp"]["send_total"] += send_delta + + if (mcdc["domain_decomp"]["busy_total"] == 0) and ( + mcdc["domain_decomp"]["send_total"] == 0 + ): + dd_signal_halt(mcdc) + + +@njit +def dd_recv_halt(mcdc): + + with objmode(): + dummy_buff = np.zeros((1,), dtype=np.int32) + MPI.COMM_WORLD.Recv(dummy_buff) + work_done = 1 + rank = MPI.COMM_WORLD.Get_rank() + + mcdc["domain_decomp"]["work_done"] = True + + +@njit +def dd_recv(mcdc): + + if mcdc["domain_decomp"]["rank_busy"]: + dd_signal_block(mcdc) + mcdc["domain_decomp"]["rank_busy"] = False + + if not mcdc["domain_decomp"]["work_done"]: + tag = dd_get_recv_tag() + + if tag == 1: + dd_recv_particles(mcdc) + elif tag == 2: + dd_recv_turnstile(mcdc) + elif tag == 3: + dd_recv_halt(mcdc) # ============================================================================= diff --git a/mcdc/loop.py b/mcdc/loop.py index f5921e2e..ef3c9a46 100644 --- a/mcdc/loop.py +++ b/mcdc/loop.py @@ -3,6 +3,8 @@ from numba import njit, objmode, jit from scipy.linalg import eig +from mpi4py import MPI + import mcdc.kernel as kernel import mcdc.type_ as type_ import pathlib @@ -151,6 +153,9 @@ def loop_source(seed, mcdc): # Progress bar indicator N_prog = 0 + if mcdc["technique"]["domain_decomposition"]: + kernel.dd_check_in(mcdc) + # Loop over particle sources work_start = mcdc["mpi_work_start"] work_size = mcdc["mpi_work_size"] @@ -238,7 +243,10 @@ def loop_source(seed, mcdc): kernel.dd_particle_send(mcdc) terminated = False max_work = 1 - kernel.dd_particle_receive(mcdc) + kernel.dd_recv(mcdc) + if mcdc["domain_decomp"]["work_done"]: + terminated = True + while not terminated: if mcdc["bank_active"]["size"] > 0: # Loop until active bank is exhausted @@ -246,7 +254,7 @@ def loop_source(seed, mcdc): P = kernel.get_particle(mcdc["bank_active"], mcdc) if not kernel.particle_in_domain(P, mcdc) and P["alive"] == True: - print("recieved particle not in domain, position:") + print(f"recieved particle not in domain, index") # Apply weight window if mcdc["technique"]["weight_window"]: @@ -266,15 +274,10 @@ def loop_source(seed, mcdc): ): kernel.tally_closeout_history(mcdc) - # Send all domain particle banks - kernel.dd_particle_send(mcdc) + # Send all domain particle banks + kernel.dd_particle_send(mcdc) - # Check for incoming particles - kernel.dd_particle_receive(mcdc) - work_remaining = int(kernel.allreduce(mcdc["bank_active"]["size"])) - total_sent = int(kernel.allreduce(mcdc["technique"]["dd_sent"])) - if work_remaining > max_work: - max_work = work_remaining + kernel.dd_recv(mcdc) # Progress printout """ @@ -284,7 +287,8 @@ def loop_source(seed, mcdc): with objmode(): print_progress(percent, mcdc) """ - if work_remaining + total_sent == 0: + if kernel.dd_check_halt(mcdc): + kernel.dd_check_out(mcdc) terminated = True diff --git a/mcdc/main.py b/mcdc/main.py index 81bf62ec..8a2cbbb2 100644 --- a/mcdc/main.py +++ b/mcdc/main.py @@ -111,8 +111,6 @@ def run(): # Print banner, hardware configuration, and header print_banner(mcdc) - print(mcdc["setting"]["caching"]) - set_cache(mcdc["setting"]["caching"]) print_msg(" Now running TNT...") @@ -296,6 +294,8 @@ def prepare(): type_.make_type_setting(input_deck) type_.make_type_uq_tally(input_deck) type_.make_type_uq(input_deck) + type_.make_type_domain_decomp(input_deck) + type_.make_type_dd_turnstile_event(input_deck) type_.make_type_technique(input_deck) type_.make_type_global(input_deck) diff --git a/mcdc/type_.py b/mcdc/type_.py index 2e59a7f4..e2b3e7a2 100644 --- a/mcdc/type_.py +++ b/mcdc/type_.py @@ -2,6 +2,7 @@ import numpy as np from mpi4py import MPI +from mpi4py.util.dtlib import from_numpy_dtype # Basic types float64 = np.float64 @@ -79,7 +80,7 @@ def make_type_particle(input_deck): # Particle record (in-bank) def make_type_particle_record(input_deck): - global particle_record + global particle_record, particle_record_mpi struct = [ ("x", float64), @@ -115,6 +116,9 @@ def make_type_particle_record(input_deck): # Save type particle_record = np.dtype(struct) + particle_record_mpi = from_numpy_dtype(particle_record) + particle_record_mpi.Commit() + precursor = np.dtype( [ @@ -1023,6 +1027,62 @@ def make_type_parameter(G, J, decay=False): ) +def make_type_dd_turnstile_event(input_deck): + global dd_turnstile_event, dd_turnstile_event_mpi + dd_turnstile_event = np.dtype( + [ + ("busy_delta", int32), + ("send_delta", int32), + ] + ) + dd_turnstile_event_mpi = from_numpy_dtype(dd_turnstile_event) + dd_turnstile_event_mpi.Commit() + + +def make_type_domain_decomp(input_deck): + global domain_decomp + # Domain banks if needed + if input_deck.technique["domain_decomposition"]: + bank_domain_xp = particle_bank(input_deck.technique["dd_exchange_rate"]) + bank_domain_xn = particle_bank(input_deck.technique["dd_exchange_rate"]) + bank_domain_yp = particle_bank(input_deck.technique["dd_exchange_rate"]) + bank_domain_yn = particle_bank(input_deck.technique["dd_exchange_rate"]) + bank_domain_zp = particle_bank(input_deck.technique["dd_exchange_rate"]) + bank_domain_zn = particle_bank(input_deck.technique["dd_exchange_rate"]) + else: + bank_domain_xp = particle_bank(0) + bank_domain_xn = particle_bank(0) + bank_domain_yp = particle_bank(0) + bank_domain_yn = particle_bank(0) + bank_domain_zp = particle_bank(0) + bank_domain_zn = particle_bank(0) + + domain_decomp = np.dtype( + [ + # Info tracked in all ranks + ("bank_xp", bank_domain_xp), + ("bank_xn", bank_domain_xn), + ("bank_yp", bank_domain_yp), + ("bank_yn", bank_domain_yn), + ("bank_zp", bank_domain_zp), + ("bank_zn", bank_domain_zn), + ("send_count", int64), # Number of particles sent + ("recv_count", int64), # Number of particles recv'd + ("rank_busy", bool_), # True if the rank currently has particles to process + ( + "work_done", + int64, + ), # Whether or not there is any outstanding work across any ranks + # Info tracked in "leader" rank zero + ( + "send_total", + int64, + ), # The total number of particles sent but not yet recv'd + ("busy_total", int64), # The total number of busy ranks + ] + ) + + param_names = ["tag", "ID", "key", "mean", "delta", "distribution", "rng_seed"] @@ -1076,22 +1136,6 @@ def make_type_global(input_deck): bank_source = particle_bank(0) bank_precursor = precursor_bank(0) - # Domain banks if needed - if input_deck.technique["domain_decomposition"]: - bank_domain_xp = particle_bank(input_deck.technique["domain_bank_size"]) - bank_domain_xn = particle_bank(input_deck.technique["domain_bank_size"]) - bank_domain_yp = particle_bank(input_deck.technique["domain_bank_size"]) - bank_domain_yn = particle_bank(input_deck.technique["domain_bank_size"]) - bank_domain_zp = particle_bank(input_deck.technique["domain_bank_size"]) - bank_domain_zn = particle_bank(input_deck.technique["domain_bank_size"]) - else: - bank_domain_xp = particle_bank(0) - bank_domain_xn = particle_bank(0) - bank_domain_yp = particle_bank(0) - bank_domain_yn = particle_bank(0) - bank_domain_zp = particle_bank(0) - bank_domain_zn = particle_bank(0) - bank_lost = particle_bank(0) # Particle tracker N_track = 0 if input_deck.setting["track_particle"]: @@ -1129,15 +1173,10 @@ def make_type_global(input_deck): ("tally", tally), ("setting", setting), ("technique", technique), + ("domain_decomp", domain_decomp), ("bank_active", bank_active), ("bank_census", bank_census), ("bank_source", bank_source), - ("bank_domain_xp", bank_domain_xp), - ("bank_domain_xn", bank_domain_xn), - ("bank_domain_yp", bank_domain_yp), - ("bank_domain_yn", bank_domain_yn), - ("bank_domain_zp", bank_domain_zp), - ("bank_domain_zn", bank_domain_zn), ("bank_precursor", bank_precursor), ("dd_idx", int64), ("k_eff", float64), diff --git a/test/regression/run.py b/test/regression/run.py index 9ca9e6f5..0d4244d5 100644 --- a/test/regression/run.py +++ b/test/regression/run.py @@ -111,7 +111,7 @@ answer = h5py.File("answer.h5", "r") runtimes[-1] = output["runtime/total"][()] - print(" (%.2f seconds)" % runtimes[-1]) + print(" (%.2f seconds)" % runtimes[-1][0]) # Compare all scores for score in [key for key in output["tally"].keys() if key != "grid"]: