From b43b8b14d6fcaf9a75208a307d837fd6466d617b Mon Sep 17 00:00:00 2001 From: Corentin Date: Sat, 23 Jan 2021 04:18:05 +0900 Subject: [PATCH 01/19] Trainer verbose argument implementation --- trainer.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/trainer.py b/trainer.py index 92b8f8f..39742ce 100644 --- a/trainer.py +++ b/trainer.py @@ -23,12 +23,13 @@ class Trainer: epoch_skip: int, summary_per_epoch: int, image_per_epoch: int, data_dtype=None, label_dtype=None, train_pre_process: Optional[nn.Module] = None, data_is_label: bool = False, - logger=DummyLogger()): + logger=DummyLogger(), verbose=True): super().__init__() self.device = device self.output_dir = output_dir self.data_is_label = data_is_label self.logger = logger + self.verbose = verbose self.should_stop = False self.batch_generator_train = batch_generator_train @@ -129,16 +130,17 @@ class Trainer: try: while not self.should_stop and self.batch_generator_train.epoch < epochs: epoch = self.batch_generator_train.epoch - print() - print(' ' * os.get_terminal_size()[0], end='\r') - print(f'Epoch {self.batch_generator_train.epoch}') + if self.verbose: + print() + print(' ' * os.get_terminal_size()[0], end='\r') + print(f'Epoch {self.batch_generator_train.epoch}') while not self.should_stop and epoch == self.batch_generator_train.epoch: self.batch_inputs = torch.as_tensor( self.batch_generator_train.batch_data, dtype=self.data_dtype, device=self.device) self.batch_labels = torch.as_tensor( self.batch_generator_train.batch_label, dtype=self.label_dtype, device=self.device) - if self.benchmark_step > 1: + if self.verbose and self.benchmark_step > 1: speed = self.benchmark_step / (time.time() - self.benchmark_time) print( f'Step {self.batch_generator_train.global_step}, {speed:0.02f} steps/s' @@ -167,7 +169,8 @@ class Trainer: self.save_summaries() self.batch_generator_train.next_batch() except KeyboardInterrupt: - print() + if self.verbose: + print() train_stop_time = time.time() self.writer_train.close() @@ -228,10 +231,11 @@ class Trainer: self.batch_inputs, self.processed_inputs, self.batch_labels, self.network_outputs, val_inputs, val_pre_process, val_labels, val_outputs) - speed = self.benchmark_step / (time.time() - self.benchmark_time) - print(f'Step {global_step}, ' - f'loss {self.running_loss / self.running_count:.03e} {val_loss / val_count:.03e}, ' - f'{speed:0.02f} steps/s, {speed * self.batch_generator_train.batch_size:0.02f} input/sec') + if self.verbose: + speed = self.benchmark_step / (time.time() - self.benchmark_time) + print(f'Step {global_step}, ' + f'loss {self.running_loss / self.running_count:.03e} {val_loss / val_count:.03e}, ' + f'{speed:0.02f} steps/s, {speed * self.batch_generator_train.batch_size:0.02f} input/sec') torch.save(self.network, os.path.join(self.output_dir, f'model_{global_step}.pt')) self.benchmark_time = time.time() self.benchmark_step = 0 From d315f342a49f96a2919199a1ddb24b65d4770e7e Mon Sep 17 00:00:00 2001 From: Corentin Date: Tue, 26 Jan 2021 01:15:02 +0900 Subject: [PATCH 02/19] Accuracy function implemented in Trainer --- trainer.py | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/trainer.py b/trainer.py index 39742ce..772cef3 100644 --- a/trainer.py +++ b/trainer.py @@ -43,6 +43,7 @@ class Trainer: self.network = network self.optimizer = optimizer self.criterion = criterion + self.accuracy_fn: Optional[Callable[[torch.Tensor, torch.Tensor], torch.Tensor]] = None self.writer_train = SummaryWriter(log_dir=os.path.join(output_dir, 'train'), flush_secs=30) self.writer_val = SummaryWriter(log_dir=os.path.join(output_dir, 'val'), flush_secs=30) @@ -90,7 +91,9 @@ class Trainer: self.processed_inputs = processed_inputs self.network_outputs = processed_inputs # Placeholder self.train_loss = 0.0 + self.train_accuracy = 0.0 self.running_loss = 0.0 + self.running_accuracy = 0.0 self.running_count = 0 self.benchmark_step = 0 self.benchmark_time = time.time() @@ -99,14 +102,14 @@ class Trainer: self, batch_inputs: torch.Tensor, processed_inputs: torch.Tensor, batch_labels: torch.Tensor, network_outputs: torch.Tensor, - loss: float): + loss: float, accuracy: float): pass def val_step_callback( self, batch_inputs: torch.Tensor, processed_inputs: torch.Tensor, batch_labels: torch.Tensor, network_outputs: torch.Tensor, - loss: float): + loss: float, accuracy: float): pass def summary_callback( @@ -152,18 +155,20 @@ class Trainer: self.processed_inputs = self.train_pre_process(self.batch_inputs) self.network_outputs = self.network(self.processed_inputs) - loss = self.criterion( - self.network_outputs, - self.batch_labels if not self.data_is_label else self.processed_inputs) + labels = self.batch_labels if not self.data_is_label else self.processed_inputs + loss = self.criterion(self.network_outputs, labels) loss.backward() self.optimizer.step() self.train_loss = loss.item() + self.train_accuracy = self.accuracy_fn( + self.network_outputs, labels).item() if self.accuracy_fn is not None else 0.0 self.running_loss += self.train_loss + self.running_accuracy += self.train_accuracy self.running_count += len(self.batch_generator_train.batch_data) self.train_step_callback( - self.batch_inputs, self.processed_inputs, self.batch_labels, - self.network_outputs, self.train_loss) + self.batch_inputs, self.processed_inputs, labels, + self.network_outputs, self.train_loss, self.train_accuracy) self.benchmark_step += 1 self.save_summaries() @@ -192,6 +197,7 @@ class Trainer: if self.batch_generator_val.step != 0: self.batch_generator_val.skip_epoch() val_loss = 0.0 + val_accuracy = 0.0 val_count = 0 self.network.train(False) with torch.no_grad(): @@ -204,13 +210,15 @@ class Trainer: val_pre_process = self.pre_process(val_inputs) val_outputs = self.network(val_pre_process) - loss = self.criterion( - val_outputs, - val_labels if not self.data_is_label else val_pre_process).item() + val_labels = val_labels if not self.data_is_label else val_pre_process + loss = self.criterion(val_outputs, val_labels).item() + accuracy = self.accuracy_fn( + val_outputs, val_labels).item() if self.accuracy_fn is not None else 0.0 val_loss += loss + val_accuracy += accuracy val_count += len(self.batch_generator_val.batch_data) self.val_step_callback( - val_inputs, val_pre_process, val_labels, val_outputs, loss) + val_inputs, val_pre_process, val_labels, val_outputs, loss, accuracy) self.batch_generator_val.next_batch() self.network.train(True) @@ -221,6 +229,11 @@ class Trainer: 'loss', self.running_loss / self.running_count, global_step=global_step) self.writer_val.add_scalar( 'loss', val_loss / val_count, global_step=global_step) + if self.accuracy_fn is not None: + self.writer_train.add_scalar( + 'error', 1 - (self.running_accuracy / self.running_count), global_step=global_step) + self.writer_val.add_scalar( + 'error', 1 - (val_accuracy / val_count), global_step=global_step) self.summary_callback( self.batch_inputs, self.processed_inputs, self.batch_labels, self.network_outputs, self.running_count, val_inputs, val_pre_process, val_labels, val_outputs, val_count) @@ -240,4 +253,5 @@ class Trainer: self.benchmark_time = time.time() self.benchmark_step = 0 self.running_loss = 0.0 + self.running_accuracy = 0.0 self.running_count = 0 From 976f5cef490582d767a9b30cab444f03e608354c Mon Sep 17 00:00:00 2001 From: Corentin Date: Wed, 27 Jan 2021 02:04:12 +0900 Subject: [PATCH 03/19] Fix validation writer in Trainer * Add last step summary (forced) --- trainer.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/trainer.py b/trainer.py index 772cef3..9d8c05d 100644 --- a/trainer.py +++ b/trainer.py @@ -177,8 +177,10 @@ class Trainer: if self.verbose: print() + self.save_summaries(force_summary=True) train_stop_time = time.time() self.writer_train.close() + self.writer_val.close() torch.save(self.network, os.path.join(self.output_dir, 'model_final.pt')) memory_peak, gpu_memory = resource_usage() @@ -186,7 +188,7 @@ class Trainer: f'Training time : {train_stop_time - train_start_time:.03f}s\n' f'\tRAM peak : {memory_peak // 1024} MB\n\tVRAM usage : {gpu_memory}') - def save_summaries(self): + def save_summaries(self, force_summary=False): global_step = self.batch_generator_train.global_step if self.batch_generator_train.epoch < self.epoch_skip: return @@ -224,7 +226,7 @@ class Trainer: self.network.train(True) # Add summaries - if self.batch_generator_train.step % self.summary_period == (self.summary_period - 1): + if force_summary or self.batch_generator_train.step % self.summary_period == (self.summary_period - 1): self.writer_train.add_scalar( 'loss', self.running_loss / self.running_count, global_step=global_step) self.writer_val.add_scalar( From 42ae4474dda4139aa8db0c4d75fbd8d4da44b0e9 Mon Sep 17 00:00:00 2001 From: Corentin Date: Sat, 30 Jan 2021 02:11:06 +0900 Subject: [PATCH 04/19] Fix batch norm for conv2d * Change tensorboard folders * Save root scripts in output code --- layers.py | 4 ++-- trainer.py | 13 ++++++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/layers.py b/layers.py index 10df5f2..142aac0 100644 --- a/layers.py +++ b/layers.py @@ -85,7 +85,7 @@ class Conv2d(Layer): self.batch_norm = nn.BatchNorm2d( out_channels, momentum=Layer.BATCH_NORM_MOMENTUM, - track_running_stats=not Layer.BATCH_NORM_TRAINING) if self.batch_norm else None + track_running_stats=Layer.BATCH_NORM_TRAINING) if self.batch_norm else None def forward(self, input_data: torch.Tensor) -> torch.Tensor: return super().forward(self.conv(input_data)) @@ -118,7 +118,7 @@ class Deconv2d(Layer): self.batch_norm = nn.BatchNorm2d( out_channels, momentum=Layer.BATCH_NORM_MOMENTUM, - track_running_stats=not Layer.BATCH_NORM_TRAINING) if self.batch_norm else None + track_running_stats=Layer.BATCH_NORM_TRAINING) if self.batch_norm else None def forward(self, input_data: torch.Tensor) -> torch.Tensor: return super().forward(self.deconv(input_data)) diff --git a/trainer.py b/trainer.py index 9d8c05d..ddf8c6b 100644 --- a/trainer.py +++ b/trainer.py @@ -44,8 +44,8 @@ class Trainer: self.optimizer = optimizer self.criterion = criterion self.accuracy_fn: Optional[Callable[[torch.Tensor, torch.Tensor], torch.Tensor]] = None - self.writer_train = SummaryWriter(log_dir=os.path.join(output_dir, 'train'), flush_secs=30) - self.writer_val = SummaryWriter(log_dir=os.path.join(output_dir, 'val'), flush_secs=30) + self.writer_train = SummaryWriter(log_dir=os.path.join(output_dir, 'tensorboard', 'train'), flush_secs=30) + self.writer_val = SummaryWriter(log_dir=os.path.join(output_dir, 'tensorboard', 'val'), flush_secs=30) # Save network graph batch_inputs = torch.as_tensor(batch_generator_train.batch_data[:2], dtype=data_dtype, device=device) @@ -79,7 +79,7 @@ class Trainer: torch.save(network, os.path.join(output_dir, 'model_init.pt')) # Save source files for entry in glob.glob(os.path.join('config', '**', '*.py'), recursive=True) + glob.glob( - os.path.join('src', '**', '*.py'), recursive=True): + os.path.join('src', '**', '*.py'), recursive=True) + glob.glob('*.py'): dirname = os.path.join(output_dir, 'code', os.path.dirname(entry)) if not os.path.exists(dirname): os.makedirs(dirname) @@ -181,7 +181,6 @@ class Trainer: train_stop_time = time.time() self.writer_train.close() self.writer_val.close() - torch.save(self.network, os.path.join(self.output_dir, 'model_final.pt')) memory_peak, gpu_memory = resource_usage() self.logger.info( @@ -251,7 +250,11 @@ class Trainer: print(f'Step {global_step}, ' f'loss {self.running_loss / self.running_count:.03e} {val_loss / val_count:.03e}, ' f'{speed:0.02f} steps/s, {speed * self.batch_generator_train.batch_size:0.02f} input/sec') - torch.save(self.network, os.path.join(self.output_dir, f'model_{global_step}.pt')) + if self.accuracy_fn is not None: + torch.save(self.network, os.path.join( + self.output_dir, f'step_{global_step}_acc_{val_accuracy / val_count:.04f}.pt')) + else: + torch.save(self.network, os.path.join(self.output_dir, f'step_{global_step}.pt')) self.benchmark_time = time.time() self.benchmark_step = 0 self.running_loss = 0.0 From 8ecf1752650635d736621d4a18aea0af1a12dcf0 Mon Sep 17 00:00:00 2001 From: Corentin Date: Sat, 13 Feb 2021 03:02:03 +0900 Subject: [PATCH 05/19] Trainer source saving argument * Saving network state instead of whole instance --- trainer.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/trainer.py b/trainer.py index ddf8c6b..d825e91 100644 --- a/trainer.py +++ b/trainer.py @@ -23,7 +23,7 @@ class Trainer: epoch_skip: int, summary_per_epoch: int, image_per_epoch: int, data_dtype=None, label_dtype=None, train_pre_process: Optional[nn.Module] = None, data_is_label: bool = False, - logger=DummyLogger(), verbose=True): + logger=DummyLogger(), verbose=True, save_src=True): super().__init__() self.device = device self.output_dir = output_dir @@ -76,14 +76,15 @@ class Trainer: if summary_per_epoch % image_per_epoch == 0: self.image_period = self.summary_period * (summary_per_epoch // image_per_epoch) - torch.save(network, os.path.join(output_dir, 'model_init.pt')) + torch.save(network.state_dict(), os.path.join(output_dir, 'model_init.pt')) # Save source files - for entry in glob.glob(os.path.join('config', '**', '*.py'), recursive=True) + glob.glob( - os.path.join('src', '**', '*.py'), recursive=True) + glob.glob('*.py'): - dirname = os.path.join(output_dir, 'code', os.path.dirname(entry)) - if not os.path.exists(dirname): - os.makedirs(dirname) - shutil.copy2(entry, os.path.join(output_dir, 'code', entry)) + if save_src: + for entry in glob.glob(os.path.join('config', '**', '*.py'), recursive=True) + glob.glob( + os.path.join('src', '**', '*.py'), recursive=True) + glob.glob('*.py'): + dirname = os.path.join(output_dir, 'code', os.path.dirname(entry)) + if not os.path.exists(dirname): + os.makedirs(dirname) + shutil.copy2(entry, os.path.join(output_dir, 'code', entry)) # Initialize training loop variables self.batch_inputs = batch_inputs @@ -251,10 +252,10 @@ class Trainer: f'loss {self.running_loss / self.running_count:.03e} {val_loss / val_count:.03e}, ' f'{speed:0.02f} steps/s, {speed * self.batch_generator_train.batch_size:0.02f} input/sec') if self.accuracy_fn is not None: - torch.save(self.network, os.path.join( + torch.save(self.network.state_dict(), os.path.join( self.output_dir, f'step_{global_step}_acc_{val_accuracy / val_count:.04f}.pt')) else: - torch.save(self.network, os.path.join(self.output_dir, f'step_{global_step}.pt')) + torch.save(self.network.state_dict(), os.path.join(self.output_dir, f'step_{global_step}.pt')) self.benchmark_time = time.time() self.benchmark_step = 0 self.running_loss = 0.0 From 24dea680440c59093ad7723f9efbadf8765c028f Mon Sep 17 00:00:00 2001 From: Corentin Date: Fri, 19 Feb 2021 13:35:51 +0900 Subject: [PATCH 06/19] Trainer save source parameter and batch generator fix --- utils/batch_generator.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/utils/batch_generator.py b/utils/batch_generator.py index 91b70e4..a39e396 100644 --- a/utils/batch_generator.py +++ b/utils/batch_generator.py @@ -59,6 +59,8 @@ class BatchGenerator: self.last_batch_size = len(self.index_list) % self.batch_size if self.last_batch_size == 0: self.last_batch_size = self.batch_size + else: + self.step_per_epoch += 1 self.epoch = 0 self.global_step = 0 @@ -194,10 +196,8 @@ class BatchGenerator: self.current_cache = 0 parent_cache_data = self.cache_data parent_cache_label = self.cache_label - cache_data = np.ndarray(self.cache_data[0].shape, dtype=self.cache_data[0].dtype) - cache_label = np.ndarray(self.cache_label[0].shape, dtype=self.cache_label[0].dtype) - self.cache_data = [cache_data] - self.cache_label = [cache_label] + self.cache_data = [np.ndarray(self.cache_data[0].shape, dtype=self.cache_data[0].dtype)] + self.cache_label = [np.ndarray(self.cache_label[0].shape, dtype=self.cache_label[0].dtype)] self.index_list[:] = self.cache_indices pipe = self.worker_pipes[worker_index][1] @@ -208,8 +208,6 @@ class BatchGenerator: continue self.index_list = self.cache_indices[start_index:start_index + self.batch_size].copy() - self.cache_data[0] = cache_data[:self.batch_size] - self.cache_label[0] = cache_label[:self.batch_size] self._next_batch() parent_cache_data[current_cache][batch_index:batch_index + self.batch_size] = self.cache_data[ self.current_cache][:self.batch_size] @@ -338,7 +336,6 @@ if __name__ == '__main__': for _ in range(19): print(batch_generator.batch_data, batch_generator.epoch, batch_generator.step) batch_generator.next_batch() - raise KeyboardInterrupt print() test() From 50c395a07f4d91d815b72fd6c9f08438201ec62b Mon Sep 17 00:00:00 2001 From: Corentin Date: Thu, 25 Feb 2021 02:18:02 +0900 Subject: [PATCH 07/19] Trainer last summary fix + memory utils --- train.py | 21 ++++++++++++++++++--- trainer.py | 17 +++++++++++++++++ utils/memory.py | 19 +++++++++++++++++++ 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/train.py b/train.py index bfd324d..0e026d3 100644 --- a/train.py +++ b/train.py @@ -28,11 +28,26 @@ def parameter_summary(network: torch.nn.Module) -> List[Tuple[str, Tuple[int], s def resource_usage() -> Tuple[int, str]: memory_peak = int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss) + return memory_peak, gpu_used_memory() + + +def gpu_used_memory() -> str: gpu_memory = subprocess.check_output( - 'nvidia-smi --query-gpu=memory.used --format=csv,noheader', shell=True).decode() + 'nvidia-smi --query-gpu=memory.used --format=csv,noheader', shell=True).decode().strip() if 'CUDA_VISIBLE_DEVICES' in os.environ: gpu_memory = gpu_memory.split('\n')[int(os.environ['CUDA_VISIBLE_DEVICES'])] else: - gpu_memory = ' '.join(gpu_memory.split('\n')) + gpu_memory = ','.join(gpu_memory.split('\n')) - return memory_peak, gpu_memory + return gpu_memory + + +def gpu_total_memory() -> str: + gpu_memory = subprocess.check_output( + 'nvidia-smi --query-gpu=memory.total --format=csv,noheader', shell=True).decode().strip() + if 'CUDA_VISIBLE_DEVICES' in os.environ: + gpu_memory = gpu_memory.split('\n')[int(os.environ['CUDA_VISIBLE_DEVICES'])] + else: + gpu_memory = ','.join(gpu_memory.split('\n')) + + return gpu_memory diff --git a/trainer.py b/trainer.py index d825e91..bccf875 100644 --- a/trainer.py +++ b/trainer.py @@ -178,6 +178,23 @@ class Trainer: if self.verbose: print() + # Small training loop for last metrics + for _ in range(20): + self.batch_inputs = torch.as_tensor( + self.batch_generator_train.batch_data, dtype=self.data_dtype, device=self.device) + self.batch_labels = torch.as_tensor( + self.batch_generator_train.batch_label, dtype=self.label_dtype, device=self.device) + + self.processed_inputs = self.train_pre_process(self.batch_inputs) + self.network_outputs = self.network(self.processed_inputs) + + self.train_loss = loss.item() + self.train_accuracy = self.accuracy_fn( + self.network_outputs, labels).item() if self.accuracy_fn is not None else 0.0 + self.running_loss += self.train_loss + self.running_accuracy += self.train_accuracy + self.running_count += len(self.batch_generator_train.batch_data) + self.benchmark_step += 1 self.save_summaries(force_summary=True) train_stop_time = time.time() self.writer_train.close() diff --git a/utils/memory.py b/utils/memory.py index be26ffc..5a8096f 100644 --- a/utils/memory.py +++ b/utils/memory.py @@ -6,3 +6,22 @@ def human_size(byte_count: int) -> str: break amount /= 1024.0 return f'{amount:.2f}{unit}B' + + +def human_to_bytes(text: str) -> float: + split_index = 0 + while '0' <= text[split_index] <= '9': + split_index += 1 + if split_index == len(text): + return float(text) + amount = float(text[:split_index]) + unit = text[split_index:].strip() + + if not unit: + return amount + if unit not in ['KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB']: + raise RuntimeError(f'Unrecognized unit : {unit}') + for final_unit in ['KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB']: + amount *= 1024.0 + if unit == final_unit: + return amount From 1fe6cd796e2a6f42563f9b52dc6ed3399e5eb711 Mon Sep 17 00:00:00 2001 From: Corentin Date: Mon, 1 Mar 2021 23:56:15 +0900 Subject: [PATCH 08/19] Fix last training loop --- trainer.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/trainer.py b/trainer.py index bccf875..4f94396 100644 --- a/trainer.py +++ b/trainer.py @@ -187,6 +187,8 @@ class Trainer: self.processed_inputs = self.train_pre_process(self.batch_inputs) self.network_outputs = self.network(self.processed_inputs) + labels = self.batch_labels if not self.data_is_label else self.processed_inputs + loss = self.criterion(self.network_outputs, labels) self.train_loss = loss.item() self.train_accuracy = self.accuracy_fn( @@ -195,6 +197,7 @@ class Trainer: self.running_accuracy += self.train_accuracy self.running_count += len(self.batch_generator_train.batch_data) self.benchmark_step += 1 + self.batch_generator_train.next_batch() self.save_summaries(force_summary=True) train_stop_time = time.time() self.writer_train.close() From 92971be5f0ab91ee35f3cbd53f14d7166d562938 Mon Sep 17 00:00:00 2001 From: Corentin Date: Wed, 10 Mar 2021 08:43:42 +0900 Subject: [PATCH 09/19] Pre summary callback added to Trainer --- trainer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/trainer.py b/trainer.py index 4f94396..1bb7352 100644 --- a/trainer.py +++ b/trainer.py @@ -113,6 +113,9 @@ class Trainer: loss: float, accuracy: float): pass + def pre_summary_callback(self): + pass + def summary_callback( self, train_inputs: torch.Tensor, train_processed: torch.Tensor, @@ -218,6 +221,7 @@ class Trainer: if self.batch_generator_val.step != 0: self.batch_generator_val.skip_epoch() + self.pre_summary_callback() val_loss = 0.0 val_accuracy = 0.0 val_count = 0 From 51776d699941fb2d1811773a36ce35a9e2f6c30e Mon Sep 17 00:00:00 2001 From: Corentin Date: Thu, 18 Mar 2021 10:36:17 +0900 Subject: [PATCH 10/19] Sequence stride option for SequenceBatchGenerator --- utils/sequence_batch_generator.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/utils/sequence_batch_generator.py b/utils/sequence_batch_generator.py index 04cd55e..041bd60 100644 --- a/utils/sequence_batch_generator.py +++ b/utils/sequence_batch_generator.py @@ -17,7 +17,7 @@ class SequenceGenerator(BatchGenerator): def __init__(self, data: Iterable, label: Iterable, sequence_size: int, batch_size: int, data_processor: Optional[Callable] = None, label_processor: Optional[Callable] = None, prefetch=True, preload=False, num_workers=1, shuffle=True, initial_shuffle=False, - flip_data=False, save: Optional[str] = None): + sequence_stride=1, flip_data=False, save: Optional[str] = None): self.batch_size = batch_size self.sequence_size = sequence_size self.shuffle = shuffle @@ -72,9 +72,14 @@ class SequenceGenerator(BatchGenerator): self.index_list = [] for sequence_index in range(len(self.data)): - start_indices = np.expand_dims( - np.arange(len(self.data[sequence_index]) - sequence_size + 1, dtype=np.uint32), - axis=-1) + if sequence_stride > 1: + start_indices = np.expand_dims( + np.arange(0, len(self.data[sequence_index]) - sequence_size + 1, sequence_stride, dtype=np.uint32), + axis=-1) + else: + start_indices = np.expand_dims( + np.arange(len(self.data[sequence_index]) - sequence_size + 1, dtype=np.uint32), + axis=-1) start_indices = np.insert(start_indices, 0, sequence_index, axis=1) self.index_list.append(start_indices) self.index_list = np.concatenate(self.index_list, axis=0) From 86787f6517635acd49318363e24a4caa59771a2f Mon Sep 17 00:00:00 2001 From: Corentin Date: Thu, 18 Mar 2021 21:30:59 +0900 Subject: [PATCH 11/19] Pipeline implementation for BatchGenerator * Add end_epoch_callback in Trainer * Fix Layer.ACTIVATION in cas of nn.Module --- layers.py | 7 ++++++- trainer.py | 4 ++++ utils/batch_generator.py | 26 +++++++++++++++++--------- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/layers.py b/layers.py index 142aac0..6d511f6 100644 --- a/layers.py +++ b/layers.py @@ -33,7 +33,12 @@ class Layer(nn.Module): self.info = LayerInfo() # Preload default - self.activation = Layer.ACTIVATION if activation == 0 else activation + if activation == 0: + activation = Layer.ACTIVATION + if isinstance(activation, type): + self.activation = activation() + else: + self.activation = activation self.batch_norm = Layer.BATCH_NORM if batch_norm is None else batch_norm def forward(self, input_data: torch.Tensor) -> torch.Tensor: diff --git a/trainer.py b/trainer.py index 1bb7352..4059b9f 100644 --- a/trainer.py +++ b/trainer.py @@ -99,6 +99,9 @@ class Trainer: self.benchmark_step = 0 self.benchmark_time = time.time() + def end_epoch_callback(self): + pass + def train_step_callback( self, batch_inputs: torch.Tensor, processed_inputs: torch.Tensor, @@ -177,6 +180,7 @@ class Trainer: self.benchmark_step += 1 self.save_summaries() self.batch_generator_train.next_batch() + self.end_epoch_callback() except KeyboardInterrupt: if self.verbose: print() diff --git a/utils/batch_generator.py b/utils/batch_generator.py index a39e396..cfedc4c 100644 --- a/utils/batch_generator.py +++ b/utils/batch_generator.py @@ -11,6 +11,7 @@ class BatchGenerator: def __init__(self, data: Iterable, label: Iterable, batch_size: int, data_processor: Optional[Callable] = None, label_processor: Optional[Callable] = None, + pipeline: Optional[Callable] = None, prefetch=True, preload=False, num_workers=1, shuffle=True, initial_shuffle=False, flip_data=False, save: Optional[str] = None): self.batch_size = batch_size @@ -18,6 +19,7 @@ class BatchGenerator: self.prefetch = prefetch and not preload self.num_workers = num_workers self.flip_data = flip_data + self.pipeline = pipeline if not preload: self.data_processor = data_processor @@ -261,31 +263,37 @@ class BatchGenerator: data = [] for entry in self.index_list[self.step * self.batch_size:(self.step + 1) * self.batch_size]: data.append(self.data_processor(self.data[entry])) - self.cache_data[self.current_cache][:len(data)] = np.asarray(data) + data = np.asarray(data) else: data = self.data[ self.index_list[self.step * self.batch_size: (self.step + 1) * self.batch_size]] if self.flip_data: flip = np.random.uniform() if flip < 0.25: - self.cache_data[self.current_cache][:len(data)] = data[:, :, ::-1] + data = data[:, :, ::-1] elif flip < 0.5: - self.cache_data[self.current_cache][:len(data)] = data[:, :, :, ::-1] + data = data[:, :, :, ::-1] elif flip < 0.75: - self.cache_data[self.current_cache][:len(data)] = data[:, :, ::-1, ::-1] - else: - self.cache_data[self.current_cache][:len(data)] = data - else: - self.cache_data[self.current_cache][:len(data)] = data + data = data[:, :, ::-1, ::-1] + # Loading label if self.label_processor is not None: label = [] for entry in self.index_list[self.step * self.batch_size:(self.step + 1) * self.batch_size]: label.append(self.label_processor(self.label[entry])) - self.cache_label[self.current_cache][:len(label)] = np.asarray(label) + label = np.asarray(label) else: label = self.label[ self.index_list[self.step * self.batch_size: (self.step + 1) * self.batch_size]] + + # Process through pipeline + if self.pipeline is not None: + for data_index, data_entry in enumerate(data): + piped_data, piped_label = self.pipeline(data_entry, label[data_index]) + self.cache_data[self.current_cache][data_index] = piped_data + self.cache_label[self.current_cache][data_index] = piped_label + else: + self.cache_data[self.current_cache][:len(data)] = data self.cache_label[self.current_cache][:len(label)] = label def next_batch(self) -> Tuple[np.ndarray, np.ndarray]: From d0fd6b2642ddf2e6010c60ff8cead8495e70ffc3 Mon Sep 17 00:00:00 2001 From: Corentin Date: Mon, 22 Mar 2021 20:14:27 +0900 Subject: [PATCH 12/19] Index list parameter for SequenceBatchGenerator --- utils/sequence_batch_generator.py | 35 ++++++++++++++++++------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/utils/sequence_batch_generator.py b/utils/sequence_batch_generator.py index 041bd60..7affb81 100644 --- a/utils/sequence_batch_generator.py +++ b/utils/sequence_batch_generator.py @@ -16,7 +16,7 @@ class SequenceGenerator(BatchGenerator): def __init__(self, data: Iterable, label: Iterable, sequence_size: int, batch_size: int, data_processor: Optional[Callable] = None, label_processor: Optional[Callable] = None, - prefetch=True, preload=False, num_workers=1, shuffle=True, initial_shuffle=False, + index_list=None, prefetch=True, preload=False, num_workers=1, shuffle=True, initial_shuffle=False, sequence_stride=1, flip_data=False, save: Optional[str] = None): self.batch_size = batch_size self.sequence_size = sequence_size @@ -70,19 +70,26 @@ class SequenceGenerator(BatchGenerator): h5_file.create_dataset(f'data_{sequence_index}', data=self.data[sequence_index]) h5_file.create_dataset(f'label_{sequence_index}', data=self.label[sequence_index]) - self.index_list = [] - for sequence_index in range(len(self.data)): - if sequence_stride > 1: - start_indices = np.expand_dims( - np.arange(0, len(self.data[sequence_index]) - sequence_size + 1, sequence_stride, dtype=np.uint32), - axis=-1) - else: - start_indices = np.expand_dims( - np.arange(len(self.data[sequence_index]) - sequence_size + 1, dtype=np.uint32), - axis=-1) - start_indices = np.insert(start_indices, 0, sequence_index, axis=1) - self.index_list.append(start_indices) - self.index_list = np.concatenate(self.index_list, axis=0) + if index_list is not None: + self.index_list = index_list + else: + self.index_list = [] + for sequence_index in range(len(self.data)): + if sequence_stride > 1: + start_indices = np.expand_dims( + np.arange(0, + len(self.data[sequence_index]) - sequence_size + 1, + sequence_stride, + dtype=np.uint32), + axis=-1) + else: + start_indices = np.expand_dims( + np.arange(len(self.data[sequence_index]) - sequence_size + 1, dtype=np.uint32), + axis=-1) + start_indices = np.insert(start_indices, 0, sequence_index, axis=1) + self.index_list.append(start_indices) + self.index_list = np.concatenate(self.index_list, axis=0) + if shuffle or initial_shuffle: np.random.shuffle(self.index_list) self.step_per_epoch = len(self.index_list) // self.batch_size From 73457750a8ca0be957fb0ffbca4a15c5a71d3b2b Mon Sep 17 00:00:00 2001 From: Corentin Date: Tue, 23 Mar 2021 02:13:40 +0900 Subject: [PATCH 13/19] Add pipeline to SequenceBatchGenerator * Remove flip_data parameter (can be done in the pipeline) --- utils/batch_generator.py | 13 +++++--- utils/sequence_batch_generator.py | 54 +++++++++++++------------------ 2 files changed, 31 insertions(+), 36 deletions(-) diff --git a/utils/batch_generator.py b/utils/batch_generator.py index cfedc4c..b8257ef 100644 --- a/utils/batch_generator.py +++ b/utils/batch_generator.py @@ -68,10 +68,15 @@ class BatchGenerator: self.global_step = 0 self.step = 0 - first_data = np.array([data_processor(entry) if data_processor else entry - for entry in self.data[self.index_list[:batch_size]]]) - first_label = np.array([label_processor(entry) if label_processor else entry - for entry in self.label[self.index_list[:batch_size]]]) + first_data = [data_processor(entry) if data_processor else entry + for entry in self.data[self.index_list[:batch_size]]] + first_label = [label_processor(entry) if label_processor else entry + for entry in self.label[self.index_list[:batch_size]]] + if self.pipeline is not None: + for data_index, sample_data in enumerate(first_data): + first_data[data_index], first_label[data_index] = self.pipeline(sample_data, first_label[data_index]) + first_data = np.asarray(first_data) + first_label = np.asarray(first_label) self.batch_data = first_data self.batch_label = first_label diff --git a/utils/sequence_batch_generator.py b/utils/sequence_batch_generator.py index 7affb81..524e58b 100644 --- a/utils/sequence_batch_generator.py +++ b/utils/sequence_batch_generator.py @@ -16,14 +16,15 @@ class SequenceGenerator(BatchGenerator): def __init__(self, data: Iterable, label: Iterable, sequence_size: int, batch_size: int, data_processor: Optional[Callable] = None, label_processor: Optional[Callable] = None, - index_list=None, prefetch=True, preload=False, num_workers=1, shuffle=True, initial_shuffle=False, - sequence_stride=1, flip_data=False, save: Optional[str] = None): + pipeline: Optional[Callable] = None, index_list=None, + prefetch=True, preload=False, num_workers=1, shuffle=True, initial_shuffle=False, + sequence_stride=1, save: Optional[str] = None): self.batch_size = batch_size self.sequence_size = sequence_size self.shuffle = shuffle self.prefetch = prefetch and not preload self.num_workers = num_workers - self.flip_data = flip_data + self.pipeline = pipeline if not preload: self.data_processor = data_processor @@ -107,26 +108,29 @@ class SequenceGenerator(BatchGenerator): first_data.append( [data_processor(input_data) for input_data in self.data[sequence_index][start_index: start_index + self.sequence_size]]) - first_data = np.asarray(first_data) else: first_data = [] for sequence_index, start_index in self.index_list[:batch_size]: first_data.append( self.data[sequence_index][start_index: start_index + self.sequence_size]) - first_data = np.asarray(first_data) if label_processor: first_label = [] for sequence_index, start_index in self.index_list[:batch_size]: first_label.append( [label_processor(input_label) for input_label in self.label[sequence_index][start_index: start_index + self.sequence_size]]) - first_label = np.asarray(first_label) else: first_label = [] for sequence_index, start_index in self.index_list[:batch_size]: first_label.append( self.label[sequence_index][start_index: start_index + self.sequence_size]) - first_label = np.asarray(first_label) + if self.pipeline is not None: + for sequence_index, (data_sequence, label_sequence) in enumerate(zip(first_data, first_label)): + for data_index, (data_sample, label_sample) in enumerate(zip(data_sequence, label_sequence)): + first_data[sequence_index][data_index], first_label[sequence_index][data_index] = self.pipeline( + data_sample, label_sample) + first_data = np.asarray(first_data) + first_label = np.asarray(first_label) self.batch_data = first_data self.batch_label = first_label @@ -179,35 +183,11 @@ class SequenceGenerator(BatchGenerator): data.append( [self.data_processor(input_data) for input_data in self.data[sequence_index][start_index: start_index + self.sequence_size]]) - if self.flip_data: - flip = np.random.uniform() - if flip < 0.25: - self.cache_data[self.current_cache][:len(data)] = np.asarray(data)[:, :, :, ::-1] - elif flip < 0.5: - self.cache_data[self.current_cache][:len(data)] = np.asarray(data)[:, :, :, :, ::-1] - elif flip < 0.75: - self.cache_data[self.current_cache][:len(data)] = np.asarray(data)[:, :, :, ::-1, ::-1] - else: - self.cache_data[self.current_cache][:len(data)] = np.asarray(data) - else: - self.cache_data[self.current_cache][:len(data)] = np.asarray(data) else: data = [] for sequence_index, start_index in self.index_list[ self.step * self.batch_size:(self.step + 1) * self.batch_size]: data.append(self.data[sequence_index][start_index: start_index + self.sequence_size]) - if self.flip_data: - flip = np.random.uniform() - if flip < 0.25: - self.cache_data[self.current_cache][:len(data)] = np.asarray(data)[:, :, :, ::-1] - elif flip < 0.5: - self.cache_data[self.current_cache][:len(data)] = np.asarray(data)[:, :, :, :, ::-1] - elif flip < 0.75: - self.cache_data[self.current_cache][:len(data)] = np.asarray(data)[:, :, :, ::-1, ::-1] - else: - self.cache_data[self.current_cache][:len(data)] = np.asarray(data) - else: - self.cache_data[self.current_cache][:len(data)] = np.asarray(data) # Loading label if self.label_processor is not None: @@ -217,12 +197,22 @@ class SequenceGenerator(BatchGenerator): label.append( [self.label_processor(input_data) for input_data in self.label[sequence_index][start_index: start_index + self.sequence_size]]) - self.cache_label[self.current_cache][:len(label)] = np.asarray(label) else: label = [] for sequence_index, start_index in self.index_list[ self.step * self.batch_size:(self.step + 1) * self.batch_size]: label.append(self.label[sequence_index][start_index: start_index + self.sequence_size]) + + # Process through pipeline + if self.pipeline is not None: + for sequence_index in range(len(data)): + for data_index in range(len(data[sequence_index])): + piped_data, piped_label = self.pipeline( + data[sequence_index][data_index], label[sequence_index][data_index]) + self.cache_data[self.current_cache][data_index] = piped_data + self.cache_label[self.current_cache][data_index] = piped_label + else: + self.cache_data[self.current_cache][:len(data)] = np.asarray(data) self.cache_label[self.current_cache][:len(label)] = np.asarray(label) From 89240e417f8bfcb532a7d752f6fdbbc4a815e3e2 Mon Sep 17 00:00:00 2001 From: Corentin Date: Tue, 23 Mar 2021 17:36:39 +0900 Subject: [PATCH 14/19] Fix SequenceBatchGenerator's pipeline --- utils/sequence_batch_generator.py | 54 +++++++++++++++++-------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/utils/sequence_batch_generator.py b/utils/sequence_batch_generator.py index 524e58b..005caf3 100644 --- a/utils/sequence_batch_generator.py +++ b/utils/sequence_batch_generator.py @@ -125,10 +125,9 @@ class SequenceGenerator(BatchGenerator): first_label.append( self.label[sequence_index][start_index: start_index + self.sequence_size]) if self.pipeline is not None: - for sequence_index, (data_sequence, label_sequence) in enumerate(zip(first_data, first_label)): - for data_index, (data_sample, label_sample) in enumerate(zip(data_sequence, label_sequence)): - first_data[sequence_index][data_index], first_label[sequence_index][data_index] = self.pipeline( - data_sample, label_sample) + for batch_index, (data_sequence, label_sequence) in enumerate(zip(first_data, first_label)): + first_data[batch_index], first_label[batch_index] = self.pipeline( + np.asarray(data_sequence), np.asarray(label_sequence)) first_data = np.asarray(first_data) first_label = np.asarray(first_label) self.batch_data = first_data @@ -181,8 +180,9 @@ class SequenceGenerator(BatchGenerator): for sequence_index, start_index in self.index_list[ self.step * self.batch_size:(self.step + 1) * self.batch_size]: data.append( - [self.data_processor(input_data) - for input_data in self.data[sequence_index][start_index: start_index + self.sequence_size]]) + np.asarray( + [self.data_processor(input_data) + for input_data in self.data[sequence_index][start_index: start_index + self.sequence_size]])) else: data = [] for sequence_index, start_index in self.index_list[ @@ -195,8 +195,9 @@ class SequenceGenerator(BatchGenerator): for sequence_index, start_index in self.index_list[ self.step * self.batch_size:(self.step + 1) * self.batch_size]: label.append( - [self.label_processor(input_data) - for input_data in self.label[sequence_index][start_index: start_index + self.sequence_size]]) + np.asarray( + [self.label_processor(input_data) + for input_data in self.label[sequence_index][start_index: start_index + self.sequence_size]])) else: label = [] for sequence_index, start_index in self.index_list[ @@ -205,12 +206,10 @@ class SequenceGenerator(BatchGenerator): # Process through pipeline if self.pipeline is not None: - for sequence_index in range(len(data)): - for data_index in range(len(data[sequence_index])): - piped_data, piped_label = self.pipeline( - data[sequence_index][data_index], label[sequence_index][data_index]) - self.cache_data[self.current_cache][data_index] = piped_data - self.cache_label[self.current_cache][data_index] = piped_label + for batch_index in range(len(data)): + piped_data, piped_label = self.pipeline(data[batch_index], label[batch_index]) + self.cache_data[self.current_cache][batch_index] = piped_data + self.cache_label[self.current_cache][batch_index] = piped_label else: self.cache_data[self.current_cache][:len(data)] = np.asarray(data) self.cache_label[self.current_cache][:len(label)] = np.asarray(label) @@ -221,17 +220,22 @@ if __name__ == '__main__': data = np.array( [[1, 2, 3, 4, 5, 6, 7, 8, 9], [11, 12, 13, 14, 15, 16, 17, 18, 19]], dtype=np.uint8) label = np.array( - [[.1, .2, .3, .4, .5, .6, .7, .8, .9], [.11, .12, .13, .14, .15, .16, .17, .18, .19]], dtype=np.uint8) + [[10, 20, 30, 40, 50, 60, 70, 80, 90], [110, 120, 130, 140, 150, 160, 170, 180, 190]], dtype=np.uint8) - for data_processor in [None, lambda x:x]: - for prefetch in [False, True]: - for num_workers in [1, 2]: - print(f'{data_processor=} {prefetch=} {num_workers=}') - with SequenceGenerator(data, label, 5, 2, data_processor=data_processor, - prefetch=prefetch, num_workers=num_workers) as batch_generator: - for _ in range(9): - print(batch_generator.batch_data, batch_generator.epoch, batch_generator.step) - batch_generator.next_batch() - print() + def pipeline(data, label): + return data, label + + for pipeline in [None, pipeline]: + for data_processor in [None, lambda x:x]: + for prefetch in [False, True]: + for num_workers in [1, 2]: + print(f'{pipeline=} {data_processor=} {prefetch=} {num_workers=}') + with SequenceGenerator(data, label, 5, 2, data_processor=data_processor, pipeline=pipeline, + prefetch=prefetch, num_workers=num_workers) as batch_generator: + for _ in range(9): + print(batch_generator.batch_data.tolist(), batch_generator.batch_label.tolist(), + batch_generator.epoch, batch_generator.step) + batch_generator.next_batch() + print() test() From dcd99a55ea464e12f9be133ef95803927bc6c7fa Mon Sep 17 00:00:00 2001 From: Corentin Date: Mon, 26 Apr 2021 17:01:24 +0900 Subject: [PATCH 15/19] Add IPCBatchGenerator --- utils/ipc_data_generator.py | 139 ++++++++++++++++++++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 utils/ipc_data_generator.py diff --git a/utils/ipc_data_generator.py b/utils/ipc_data_generator.py new file mode 100644 index 0000000..a821c9f --- /dev/null +++ b/utils/ipc_data_generator.py @@ -0,0 +1,139 @@ +import multiprocessing as mp +from multiprocessing import shared_memory +from typing import Callable, Optional, Tuple + +import numpy as np + + +class IPCBatchGenerator: + + def __init__(self, ipc_processor: Callable, + data_processor: Optional[Callable] = None, label_processor: Optional[Callable] = None, + pipeline: Optional[Callable] = None, + prefetch=True, flip_data=False): + self.flip_data = flip_data + self.pipeline = pipeline + self.prefetch = prefetch + + self.ipc_processor = ipc_processor + self.data_processor = data_processor + self.label_processor = label_processor + + self.global_step = 0 + + self.data, self.label = ipc_processor() + first_data = [data_processor(entry) for entry in self.data] if data_processor else self.data + first_label = [label_processor(entry) for entry in self.label] if label_processor else self.label + if self.pipeline is not None: + for data_index, sample_data in enumerate(first_data): + first_data[data_index], first_label[data_index] = self.pipeline(sample_data, first_label[data_index]) + first_data = np.asarray(first_data) + first_label = np.asarray(first_label) + self.batch_data = first_data + self.batch_label = first_label + + self.process_id = 'NA' + if self.prefetch: + self.cache_memory_data = [ + shared_memory.SharedMemory(create=True, size=first_data.nbytes), + shared_memory.SharedMemory(create=True, size=first_data.nbytes)] + self.cache_data = [ + np.ndarray(first_data.shape, dtype=first_data.dtype, buffer=self.cache_memory_data[0].buf), + np.ndarray(first_data.shape, dtype=first_data.dtype, buffer=self.cache_memory_data[1].buf)] + self.cache_memory_label = [ + shared_memory.SharedMemory(create=True, size=first_label.nbytes), + shared_memory.SharedMemory(create=True, size=first_label.nbytes)] + self.cache_label = [ + np.ndarray(first_label.shape, dtype=first_label.dtype, buffer=self.cache_memory_label[0].buf), + np.ndarray(first_label.shape, dtype=first_label.dtype, buffer=self.cache_memory_label[1].buf)] + self.prefetch_pipe_parent, self.prefetch_pipe_child = mp.Pipe() + self.prefetch_stop = shared_memory.SharedMemory(create=True, size=1) + self.prefetch_stop.buf[0] = 0 + self.prefetch_process = mp.Process(target=self._prefetch_worker) + self.prefetch_process.start() + else: + self.cache_data = [first_data] + self.cache_label = [first_label] + + self.current_cache = 0 + self.process_id = 'main' + + def __del__(self): + self.release() + + def __enter__(self): + return self + + def __exit__(self, _exc_type, _exc_value, _traceback): + self.release() + + def release(self): + if self.prefetch: + self.prefetch_stop.buf[0] = 1 + self.prefetch_pipe_parent.send(True) + self.prefetch_process.join() + + for shared_mem in self.cache_memory_data + self.cache_memory_label: + shared_mem.close() + shared_mem.unlink() + self.prefetch_stop.close() + self.prefetch_stop.unlink() + self.prefetch = False # Avoids double release + + def _prefetch_worker(self): + self.prefetch = False + self.current_cache = 1 + self.process_id = 'prefetch' + + while self.prefetch_stop.buf is not None and self.prefetch_stop.buf[0] == 0: + try: + self.current_cache = 1 - self.current_cache + self.global_step += 1 + + self._next_batch() + self.prefetch_pipe_child.recv() + self.prefetch_pipe_child.send(self.current_cache) + except KeyboardInterrupt: + break + + def _next_batch(self): + # Loading data + self.data, self.label = self.ipc_processor() + + data = np.asarray([self.data_processor(entry) for entry in self.data]) if self.data_processor else self.data + if self.flip_data: + flip = np.random.uniform() + if flip < 0.25: + data = data[:, :, ::-1] + elif flip < 0.5: + data = data[:, :, :, ::-1] + elif flip < 0.75: + data = data[:, :, ::-1, ::-1] + + # Loading label + label = np.asarray([ + self.label_processor(entry) for entry in self.label]) if self.label_processor else self.label + + # Process through pipeline + if self.pipeline is not None: + for data_index, data_entry in enumerate(data): + piped_data, piped_label = self.pipeline(data_entry, label[data_index]) + self.cache_data[self.current_cache][data_index] = piped_data + self.cache_label[self.current_cache][data_index] = piped_label + else: + self.cache_data[self.current_cache][:len(data)] = data + self.cache_label[self.current_cache][:len(label)] = label + + def next_batch(self) -> Tuple[np.ndarray, np.ndarray]: + self.global_step += 1 + + if self.prefetch: + self.prefetch_pipe_parent.send(True) + self.current_cache = self.prefetch_pipe_parent.recv() + else: + self._next_batch() + + self.batch_data = self.cache_data[self.current_cache] + self.batch_label = self.cache_label[self.current_cache] + + return self.batch_data, self.batch_label From 9b0c8ec49df070f293473eeeeeda15905af5b4d9 Mon Sep 17 00:00:00 2001 From: Corentin Date: Tue, 27 Apr 2021 17:50:40 +0900 Subject: [PATCH 16/19] Fix IPCBatchGenerator SIGINT handling --- utils/ipc_data_generator.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/utils/ipc_data_generator.py b/utils/ipc_data_generator.py index a821c9f..c03a642 100644 --- a/utils/ipc_data_generator.py +++ b/utils/ipc_data_generator.py @@ -1,5 +1,6 @@ import multiprocessing as mp from multiprocessing import shared_memory +import signal from typing import Callable, Optional, Tuple import numpy as np @@ -85,16 +86,14 @@ class IPCBatchGenerator: self.current_cache = 1 self.process_id = 'prefetch' + signal.signal(signal.SIGINT, signal.SIG_IGN) while self.prefetch_stop.buf is not None and self.prefetch_stop.buf[0] == 0: - try: - self.current_cache = 1 - self.current_cache - self.global_step += 1 + self.current_cache = 1 - self.current_cache + self.global_step += 1 - self._next_batch() - self.prefetch_pipe_child.recv() - self.prefetch_pipe_child.send(self.current_cache) - except KeyboardInterrupt: - break + self._next_batch() + self.prefetch_pipe_child.recv() + self.prefetch_pipe_child.send(self.current_cache) def _next_batch(self): # Loading data From 8d13de5711d3cc2db3b73c1d4381bc196f89c65e Mon Sep 17 00:00:00 2001 From: Corentin Date: Thu, 29 Apr 2021 19:45:32 +0900 Subject: [PATCH 17/19] Improve ResNet layers --- residual.py | 74 ++++++++++++++++++++++------------------------------- 1 file changed, 30 insertions(+), 44 deletions(-) diff --git a/residual.py b/residual.py index bdc14c8..e35a009 100644 --- a/residual.py +++ b/residual.py @@ -3,65 +3,51 @@ from typing import Union, Tuple import torch import torch.nn as nn -from .layers import LayerInfo, Layer +from .layers import Conv2d, LayerInfo, Layer class ResBlock(Layer): - def __init__(self, in_channels: int, out_channels: int, kernel_size: int = 3, - activation=None, **kwargs): + def __init__(self, in_channels: int, out_channels: int = -1, kernel_size: int = 3, padding: int = 1, + stride: Union[int, Tuple[int, int]] = 1, activation=None, batch_norm=None, **kwargs): super().__init__(activation if activation is not None else 0, False) + self.batch_norm = None + if out_channels == -1: + out_channels = in_channels self.seq = nn.Sequential( - nn.Conv2d(in_channels, out_channels, kernel_size=kernel_size, bias=False, **kwargs), - nn.BatchNorm2d( - out_channels, - momentum=Layer.BATCH_NORM_MOMENTUM, - track_running_stats=not Layer.BATCH_NORM_TRAINING), - torch.nn.LeakyReLU(), - nn.Conv2d(out_channels, out_channels, kernel_size=kernel_size, bias=False, padding=1), - nn.BatchNorm2d( - out_channels, - momentum=Layer.BATCH_NORM_MOMENTUM, - track_running_stats=not Layer.BATCH_NORM_TRAINING)) - self.batch_norm = nn.BatchNorm2d( - out_channels, - momentum=Layer.BATCH_NORM_MOMENTUM, - track_running_stats=not Layer.BATCH_NORM_TRAINING) if self.batch_norm else None + Conv2d(in_channels, in_channels, kernel_size=kernel_size, stride=stride, padding=padding, **kwargs), + Conv2d(in_channels, out_channels, kernel_size=3, padding=1, + activation=None, batch_norm=batch_norm)) + self.residual = Conv2d(in_channels, out_channels, kernel_size=1, stride=stride, activation=None) if ( + out_channels != in_channels or stride != 1) else None def forward(self, input_data: torch.Tensor) -> torch.Tensor: + if self.residual is not None: + return super().forward(self.residual(input_data) + self.seq(input_data)) return super().forward(input_data + self.seq(input_data)) class ResBottleneck(Layer): - def __init__(self, in_channels: int, out_channels: int, planes: int = 1, kernel_size: int = 3, - stride: Union[int, Tuple[int, int]] = 1, activation=None, **kwargs): + def __init__(self, in_channels: int, out_channels: int = -1, bottleneck_channels: int = -1, kernel_size: int = 3, + stride: Union[int, Tuple[int, int]] = 1, padding=1, + activation=None, batch_norm=None, **kwargs): super().__init__(activation if activation is not None else 0, False) self.batch_norm = None + if out_channels == -1: + out_channels = in_channels + if bottleneck_channels == -1: + bottleneck_channels = in_channels // 4 self.seq = nn.Sequential( - nn.Conv2d(in_channels, out_channels, kernel_size=1, bias=False), - nn.BatchNorm2d( - out_channels, - momentum=Layer.BATCH_NORM_MOMENTUM, - track_running_stats=not Layer.BATCH_NORM_TRAINING), - torch.nn.LeakyReLU(), - nn.Conv2d(out_channels, out_channels, kernel_size=kernel_size, stride=stride, bias=False, **kwargs), - nn.BatchNorm2d( - out_channels, - momentum=Layer.BATCH_NORM_MOMENTUM, - track_running_stats=not Layer.BATCH_NORM_TRAINING), - torch.nn.LeakyReLU(), - nn.Conv2d(out_channels, planes * out_channels, kernel_size=1, bias=False), - nn.BatchNorm2d( - out_channels, - momentum=Layer.BATCH_NORM_MOMENTUM, - track_running_stats=not Layer.BATCH_NORM_TRAINING)) - self.downsample = nn.Sequential( - nn.Conv2d(in_channels, planes * out_channels, stride=stride, kernel_size=1), - nn.BatchNorm2d( - planes * out_channels, - momentum=Layer.BATCH_NORM_MOMENTUM, - track_running_stats=not Layer.BATCH_NORM_TRAINING)) + Conv2d(in_channels, bottleneck_channels, kernel_size=1), + Conv2d(bottleneck_channels, bottleneck_channels, kernel_size=kernel_size, + stride=stride, padding=padding, **kwargs), + Conv2d(bottleneck_channels, out_channels, kernel_size=1, + activation=None, batch_norm=batch_norm)) + self.residual = Conv2d(in_channels, out_channels, kernel_size=1, stride=stride, activation=None) if ( + out_channels != in_channels or stride != 1) else None def forward(self, input_data: torch.Tensor) -> torch.Tensor: - return super().forward(self.downsample(input_data) + self.seq(input_data)) + if self.residual is not None: + return super().forward(self.residual(input_data) + self.seq(input_data)) + return super().forward(input_data + self.seq(input_data)) From 092f4acc3b4d11c60b7a0a7954f6ba0588041a05 Mon Sep 17 00:00:00 2001 From: Corentin Date: Mon, 17 May 2021 21:05:15 +0900 Subject: [PATCH 18/19] Add SSD --- ssd/box.py | 86 ++++++++++++++++++++++++++++ ssd/ssd.py | 165 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 251 insertions(+) create mode 100644 ssd/box.py create mode 100644 ssd/ssd.py diff --git a/ssd/box.py b/ssd/box.py new file mode 100644 index 0000000..e255fe4 --- /dev/null +++ b/ssd/box.py @@ -0,0 +1,86 @@ +import numpy as np + + +def create_box(y_pos: float, x_pos: float, height: float, width: float) -> tuple[float, float, float, float]: + y_min, x_min, y_max, x_max = check_rectangle( + y_pos - (height / 2), x_pos - (width / 2), y_pos + (height / 2), x_pos + (width / 2)) + return (y_min + y_max) / 2, (x_min + x_max) / 2, y_max - y_min, x_max - x_min + + +def check_rectangle(y_min: float, x_min: float, y_max: float, x_max: float) -> tuple[float, float, float, float]: + if y_min < 0: + y_min = 0 + if x_min < 0: + x_min = 0 + if y_min > 1: + y_min = 1 + if x_min > 1: + x_min = 1 + if y_max < 0: + y_max = 0 + if x_max < 0: + x_max = 0 + if y_max >= 1: + y_max = 1 + if x_max >= 1: + x_max = 1 + return y_min, x_min, y_max, x_max + + +def get_boxes(predictions: np.ndarray, anchors: np.ndarray, class_index: int) -> np.ndarray: + boxes = np.zeros(anchors.shape) + boxes[:, 0] = (predictions[:, 0] * anchors[:, 2]) + anchors[:, 0] + boxes[:, 1] = (predictions[:, 1] * anchors[:, 3]) + anchors[:, 1] + boxes[:, 2] = np.exp(predictions[:, 2]) * anchors[:, 2] + boxes[:, 3] = np.exp(predictions[:, 3]) * anchors[:, 3] + boxes = np.asarray([create_box(*box) for box in boxes]) + + # return np.insert(boxes, 4, predictions[:, class_index], axis=-1) + return np.concatenate([boxes, predictions[:, class_index:class_index + 1]], axis=1) + + +def fast_nms(boxes: np.ndarray, min_iou: float) -> np.ndarray: + # if there are no boxes, return an empty list + if len(boxes) == 0: + return [] + + # initialize the list of picked indexes + pick = [] + + # grab the coordinates of the bounding boxes + y_min = boxes[:, 0] - (boxes[:, 2] / 2) + y_max = boxes[:, 0] + (boxes[:, 2] / 2) + x_min = boxes[:, 1] - (boxes[:, 3] / 2) + x_max = boxes[:, 1] + (boxes[:, 3] / 2) + scores = boxes[:, 4] + + # compute the area of the bounding boxes and sort the bounding boxes by the scores + areas = (x_max - x_min) * (y_max - y_min) + idxs = np.argsort(scores) + + # keep looping while some indexes still remain in the indexes + # list + while len(idxs) > 0: + # grab the last index in the indexes list and add the + # index value to the list of picked indexes + last = len(idxs) - 1 + i = idxs[last] + pick.append(i) + + inter_tops = np.maximum(y_min[i], y_min[idxs[:last]]) + inter_bottoms = np.minimum(y_max[i], y_max[idxs[:last]]) + inter_lefts = np.maximum(x_min[i], x_min[idxs[:last]]) + inter_rights = np.minimum(x_max[i], x_max[idxs[:last]]) + inter_areas = (inter_rights - inter_lefts) * (inter_bottoms - inter_tops) + + # compute the ratio of overlap + union_area = (areas[idxs[:last]] + areas[i]) - inter_areas + overlap = inter_areas / union_area + + # delete all indexes from the index list that have less overlap than min_iou + idxs = np.delete( + idxs, np.concatenate(([last], np.where(overlap > min_iou)[0]))) + + # return only the bounding boxes that were picked using the + # integer data type + return boxes[pick] diff --git a/ssd/ssd.py b/ssd/ssd.py new file mode 100644 index 0000000..eba064d --- /dev/null +++ b/ssd/ssd.py @@ -0,0 +1,165 @@ +import colorsys +import math + +import numpy as np +import torch +import torch.nn as nn + +from .box import check_rectangle +from ..layers import Conv2d + + +class SSD(nn.Module): + + class Detector(nn.Module): + def __init__(self, input_features: int, output_features: int): + super().__init__() + self.conv = Conv2d(input_features, output_features, kernel_size=3, padding=1, + batch_norm=False, activation=None) + self.output = None + + def forward(self, input_data: torch.Tensor) -> torch.Tensor: + self.output = self.conv(input_data).permute(0, 2, 3, 1) + return self.output + + class DetectorMerge(nn.Module): + def __init__(self, location_dimmension: int): + super().__init__() + self.location_dim = location_dimmension + + def forward(self, detector_outputs: torch.Tensor) -> torch.Tensor: + return torch.cat( + [detector_outputs[:, :, :self.location_dim], + torch.softmax(detector_outputs[:, :, self.location_dim:], dim=2)], dim=2) + + class AnchorInfo: + def __init__(self, center: tuple[float, float], size: tuple[float], + index: int, layer_index: int, map_index: tuple[int, int], color_index: int, + ratio: float, size_factor: float): + self.index = index + self.layer_index = layer_index + self.map_index = map_index + self.color_index = color_index + self.ratio = ratio + self.size_factor = size_factor + self.center = center + self.size = size + self.box = check_rectangle( + center[0] - (size[0] / 2), center[1] - (size[1] / 2), + center[0] + (size[0] / 2), center[1] + (size[1] / 2)) + + def __repr__(self): + return (f'{self.__class__.__name__}' + f'(index:{self.index}, layer:{self.layer_index}, coord:{self.map_index}' + f', center:({self.center[0]:.03f}, {self.center[1]:.03f})' + f', size:({self.size[0]:.03f}, {self.size[1]:.03f})' + f', ratio:{self.ratio:.03f}, size_factor:{self.size_factor:.03f})' + f', y:[{self.box[0]:.03f}:{self.box[2]:.03f}]' + f', x:[{self.box[1]:.03f}:{self.box[3]:.03f}])') + + def __array__(self): + return np.array([*self.center, *self.size]) + + def __init__(self, base_network: nn.Module, input_sample: torch.Tensor, classes: list[str], + location_dimmension: int, layer_channels: list[int], layer_box_ratios: list[float], layer_args: dict, + box_size_factors: list[float]): + super().__init__() + + self.location_dim = location_dimmension + self.classes = ['none'] + classes + self.class_count = len(self.classes) + self.base_input_shape = input_sample.numpy().shape[1:] + self.base_network = base_network + sample_output = base_network(input_sample) + self.base_output_shape = list(sample_output.detach().numpy().shape)[-3:] + + layer_convs: list[nn.Module] = [] + layer_detectors: list[SSD.Detector] = [] + last_feature_count = self.base_output_shape[0] + for layer_index, (output_features, kwargs) in enumerate(zip(layer_channels, layer_args)): + if 'disable' not in kwargs: + layer_convs.append(Conv2d(last_feature_count, output_features, **kwargs)) + layer_detectors.append(SSD.Detector( + last_feature_count, (self.class_count + self.location_dim) * len(layer_box_ratios[layer_index]))) + # layers.append(SSD.Layer( + # last_feature_count, output_features, + # (self.class_count + self.location_dim) * len(layer_box_ratios[layer_index]), + # **kwargs)) + last_feature_count = output_features + self.layer_convs = nn.ModuleList(layer_convs) + self.layer_detectors = nn.ModuleList(layer_detectors) + + self.merge = self.DetectorMerge(location_dimmension) + + self.anchors_numpy, self.anchor_info, self.box_colors = self._create_anchors( + sample_output, self.layer_convs, self.layer_detectors, layer_box_ratios, box_size_factors, + input_sample.shape[3] / input_sample.shape[2]) + self.anchors = torch.from_numpy(self.anchors_numpy) + + def forward(self, input_data: torch.Tensor) -> torch.Tensor: + head = self.base_network(input_data) + detector_outputs = [] + for layer_index, detector in enumerate(self.layer_detectors): + detector_out = detector(head) + detector_outputs.append(detector_out.reshape( + detector_out.size(0), -1, self.class_count + self.location_dim)) + if layer_index < len(self.layer_convs): + head = self.layer_convs[layer_index](head) + detector_outputs = torch.cat(detector_outputs, 1) + return self.merge(detector_outputs) + # base_output = self.base_network(input_data) + # head = base_output + # outputs = [] + # for layer in self.layers: + # head, detector_output = layer(head) + # outputs.append(detector_output.reshape(base_output.size(0), -1, self.class_count + self.location_dim)) + # outputs = torch.cat(outputs, 1) + # return torch.cat( + # [outputs[:, :, :self.location_dim], torch.softmax(outputs[:, :, self.location_dim:], dim=2)], dim=2) + + def _apply(self, fn): + super()._apply(fn) + self.anchors = fn(self.anchors) + return self + + @staticmethod + def _create_anchors( + base_output: torch.Tensor, layers: nn.ModuleList, detectors: nn.ModuleList, layer_box_ratios: list[float], + box_size_factors: list[float], image_ratio: float) -> tuple[np.ndarray, np.ndarray, list[np.ndarray]]: + anchors = [] + anchor_info: list[SSD.AnchorInfo] = [] + box_colors: list[np.ndarray] = [] + head = base_output + + for layer_index, detector in enumerate(detectors): + detector_output = detector(head) # detector output shape : NCRSHW (Ratio, Size) + if layer_index < len(layers): + head = layers[layer_index](head) + + detector_rows = detector_output.size()[1] + detector_cols = detector_output.size()[2] + color_index = 0 + layer_ratios = layer_box_ratios[layer_index] + for index_y in range(detector_rows): + center_y = (index_y + 0.5) / detector_rows + for index_x in range(detector_cols): + center_x = (index_x + 0.5) / detector_cols + for ratio, size_factor in zip(layer_ratios, box_size_factors): + box_colors.append((np.asarray(colorsys.hsv_to_rgb( + color_index / len(layer_ratios), 1.0, 1.0)) * 255).astype(np.uint8)) + color_index += 1 + unit_box_size = size_factor / max(detector_rows, detector_cols) + anchor_width = unit_box_size * math.sqrt(ratio / image_ratio) + anchor_height = unit_box_size / math.sqrt(ratio / image_ratio) + anchor_info.append(SSD.AnchorInfo( + (center_y, center_x), + (anchor_height, anchor_width), + len(anchors), + layer_index, + (index_y, index_x), + len(box_colors) - 1, + ratio, + size_factor + )) + anchors.append([center_y, center_x, anchor_height, anchor_width]) + return np.asarray(anchors, dtype=np.float32), anchor_info, box_colors From d87bb89e6ca4ed71a8f3104f83ab1c043c0aaa3c Mon Sep 17 00:00:00 2001 From: Corentin Date: Fri, 21 May 2021 15:14:14 +0900 Subject: [PATCH 19/19] SSDLoss implementation --- layers.py | 2 +- ssd/loss.py | 112 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 ssd/loss.py diff --git a/layers.py b/layers.py index 6d511f6..2c9fd9c 100644 --- a/layers.py +++ b/layers.py @@ -54,7 +54,7 @@ class Linear(Layer): def __init__(self, in_channels: int, out_channels: int, activation=0, batch_norm=None, **kwargs): super().__init__(activation, batch_norm) - self.fc = nn.Linear(in_channels, out_channels, **kwargs) + self.fc = nn.Linear(in_channels, out_channels, bias=not self.batch_norm, **kwargs) self.batch_norm = nn.BatchNorm1d( out_channels, momentum=Layer.BATCH_NORM_MOMENTUM, diff --git a/ssd/loss.py b/ssd/loss.py new file mode 100644 index 0000000..1b3d259 --- /dev/null +++ b/ssd/loss.py @@ -0,0 +1,112 @@ +import torch +import torch.nn as nn + + +class JacardOverlap(nn.Module): + def forward(self, anchors: torch.Tensor, labels: torch.Tensor) -> torch.Tensor: + """ + Assuming rank 2 (number of boxes, locations), location is (y, x, h, w) + Jaccard overlap : A ∩ B / A ∪ B = A ∩ B / (area(A) + area(B) - A ∩ B) + Return: + jaccard overlap: (tensor) Shape: [predictions.size(0), labels.size(0)] + """ + anchors_count = anchors.size(0) + labels_count = labels.size(0) + + # Getting coords (y_min, x_min, y_max, x_max) repeated to fill (anchor count, label count) + anchor_coords = torch.cat([ + anchors[:, :2] - (anchors[:, 2:] / 2), + anchors[:, :2] + (anchors[:, 2:] / 2)], 1).unsqueeze(1).expand(anchors_count, labels_count, 4) + label_coords = torch.cat([ + labels[:, :2] - (labels[:, 2:] / 2), + labels[:, :2] + (labels[:, 2:] / 2)], 1).unsqueeze(0).expand(anchors_count, labels_count, 4) + + mins = torch.max(anchor_coords, label_coords)[:, :, :2] + maxes = torch.min(anchor_coords, label_coords)[:, :, 2:] + + inter_coords = torch.clamp(maxes - mins, min=0) + inter_area = inter_coords[:, :, 0] * inter_coords[:, :, 1] + + anchor_areas = (anchors[:, 2] * anchors[:, 3]).unsqueeze(1).expand_as(inter_area) + label_areas = (labels[:, 2] * labels[:, 3]).unsqueeze(0).expand_as(inter_area) + + union_area = anchor_areas + label_areas - inter_area + return inter_area / union_area + + +class SSDLoss(nn.Module): + def __init__(self, anchors: torch.Tensor, label_per_image: int, + negative_mining_ratio: int, matching_iou: float, + location_dimmension: int = 4, localization_loss_weight: float = 1.0): + super().__init__() + self.anchors = anchors + self.anchor_count = anchors.size(0) + self.label_per_image = label_per_image + self.location_dimmension = location_dimmension + self.negative_mining_ratio = negative_mining_ratio + self.matching_iou = matching_iou + self.localization_loss_weight = localization_loss_weight + + self.overlap = JacardOverlap() + self.matches = [] + # self.negative_matches = [] + self.positive_class_loss = torch.Tensor() + self.negative_class_loss = torch.Tensor() + self.localization_loss = torch.Tensor() + self.class_loss = torch.Tensor() + self.final_loss = torch.Tensor() + + def forward(self, input_data: torch.Tensor, input_labels: torch.Tensor) -> torch.Tensor: + batch_size = input_data.size(0) + expanded_anchors = self.anchors[:, :4].unsqueeze(0).unsqueeze(2).expand( + batch_size, self.anchor_count, self.label_per_image, 4) + expanded_labels = input_labels[:, :, :self.location_dimmension].unsqueeze(1).expand( + batch_size, self.anchor_count, self.label_per_image, self.location_dimmension) + objective_pos = (expanded_labels[:, :, :, :2] - expanded_anchors[:, :, :, :2]) / ( + expanded_anchors[:, :, :, 2:]) + objective_size = torch.log(expanded_labels[:, :, :, 2:] / expanded_anchors[:, :, :, 2:]) + + positive_objectives = [] + positive_predictions = [] + positive_class_loss = [] + negative_class_loss = [] + self.matches = [] + # self.negative_matches = [] + for batch_index in range(batch_size): + predictions = input_data[batch_index] + labels = input_labels[batch_index] + overlaps = self.overlap(self.anchors[:, :4], labels[:, :4]) + mask = (overlaps >= self.matching_iou).long() + match_indices = torch.nonzero(mask, as_tuple=False) + self.matches.append(match_indices.detach().cpu()) + + mining_count = int(self.negative_mining_ratio * len(self.matches[-1])) + masked_prediction = predictions[:, self.location_dimmension] + torch.max(mask, dim=1)[0] + non_match_indices = torch.argsort(masked_prediction, dim=-1, descending=False)[:mining_count] + # self.negative_matches.append(non_match_indices.detach().cpu()) + + for anchor_index, label_index in match_indices: + positive_predictions.append(predictions[anchor_index]) + positive_objectives.append( + torch.cat(( + objective_pos[batch_index, anchor_index, label_index], + objective_size[batch_index, anchor_index, label_index]), dim=-1)) + positive_class_loss.append(torch.log( + predictions[anchor_index, self.location_dimmension + labels[label_index, -1].long()])) + + for anchor_index in non_match_indices: + negative_class_loss.append( + torch.log(predictions[anchor_index, self.location_dimmension])) + + if not positive_predictions: + return None + positive_predictions = torch.stack(positive_predictions) + positive_objectives = torch.stack(positive_objectives) + self.positive_class_loss = -torch.sum(torch.stack(positive_class_loss)) + self.negative_class_loss = -torch.sum(torch.stack(negative_class_loss)) + self.localization_loss = nn.functional.smooth_l1_loss( + positive_predictions[:, self.location_dimmension], + positive_objectives) + self.class_loss = self.positive_class_loss + self.negative_class_loss + self.final_loss = (self.localization_loss_weight * self.localization_loss) + self.class_loss + return self.final_loss