Commit 15faa8e9 authored by mova's avatar mova
Browse files

preprocessing: drop the last file with a different number of events, so that...

preprocessing: drop the last file with a different number of events, so that all preprocessed files have the same number of events
parent 13c64e0d
......@@ -39,14 +39,12 @@ Processing testing batches, queuing {len(data_loader.testing_chunks)} chunks."""
ifile = 0
for batch in tqdm(data_loader.qfseq):
output_file = f"{conf.path.training}/{ifile:03d}.pt"
batch_list.append(batch)
if len(batch_list) == data_loader.n_test_batches:
logger.info(f"Saving {output_file}")
torch.save(batch_list, f"{output_file}")
ifile += 1
batch_list = []
batch_list.append(batch)
logger.info(f"Saving {output_file}")
torch.save(batch_list, f"{output_file}")
# Drop the last uneven set of batches
data_loader.qfseq.stop()
exit(0)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment