From 2abe6c7953f72e28a60c5e05b9b3020205c35346 Mon Sep 17 00:00:00 2001 From: Corentin Date: Tue, 29 Jun 2021 01:47:50 +0900 Subject: [PATCH] Multiprocess linking + better messages --- umake.py | 107 ++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 75 insertions(+), 32 deletions(-) diff --git a/umake.py b/umake.py index 35392f2..efe19de 100755 --- a/umake.py +++ b/umake.py @@ -8,7 +8,6 @@ from pathlib import Path import subprocess import shutil import sys -from typing import List class Config: @@ -31,6 +30,18 @@ class Config: CPP_SOURCES = SOURCE_DIR.rglob('*.cpp') +class ConsoleColor: + """Simple shortcut to use colors in console""" + HEADER = '\033[95m' + BLUE = '\033[94m' + GREEN = '\033[92m' + ORANGE = '\033[93m' + RED = '\033[91m' + ENDCOLOR = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + + def get_hash(path: Path) -> str: with open(path, 'r') as hashing_file: hash_obj = hashlib.md5() @@ -94,36 +105,39 @@ def make(config: Config): # Running compilation processes if not os.path.exists(config.OBJECT_DIR): os.makedirs(config.OBJECT_DIR) - error_path = None + error_paths: list[tuple[Path, str]] = [] if arguments.j > 1: # Multi-process - jobs: List[subprocess.Popen] = [] + jobs: list[tuple[Path, Path, subprocess.Popen]] = [] for source_path, source_hash, cmd in todo_list: - print(cmd) - jobs.insert(0, (source_path, source_hash, subprocess.Popen(cmd, shell=True))) # FIFO style (will be poped) + print(ConsoleColor.BLUE + cmd + ConsoleColor.ENDCOLOR) + jobs.insert(0, (source_path, source_hash, subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, + shell=True))) # FIFO style (will be poped) if len(jobs) >= arguments.j: # If jobs count is maxed we wait for the oldest one to finished job_path, job_hash, oldest_job = jobs.pop() oldest_job.wait() if oldest_job.returncode != 0: - error_path = job_path + error_paths.append((job_path, oldest_job.stdout.read() + oldest_job.stderr.read())) break hash_dict[str(job_path)] = job_hash # Update hash if no error - if error_path is None: - for job_path, job_hash, job in jobs: # Wait the last jobs to finish - job.wait() - if job.returncode != 0: - error_path = job_path - break + for job_path, job_hash, job in jobs: # Wait the last jobs to finish + job.wait() + if job.returncode != 0: + error_paths.append((job_path, job.stdout.read() + job.stderr.read())) + else: hash_dict[str(job_path)] = job_hash # Update hash if no error else: # Single-process for source_path, source_hash, cmd in todo_list: print(cmd) - complete = subprocess.run(cmd, check=False, shell=True) - if complete.returncode != 0: - error_path = source_path + job = subprocess.run(cmd, check=False, stdin=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True, shell=True) + if job.returncode != 0: + error_paths.append((source_path, job.stdout.read() + job.stderr.read())) break hash_dict[str(source_path)] = source_hash # Update hash if no error - if error_path: - print(f'Error compiling {error_path}') + if error_paths: + for error_path, error_text in error_paths: + print(ConsoleColor.RED + f'Error compiling {error_path}:\n' + ConsoleColor.ENDCOLOR + error_text) with open(config.OBJECT_DIR / 'hash.json', 'w') as hash_file: hash_file.write(json.dumps(hash_dict, indent=1)) sys.exit(1) @@ -133,23 +147,52 @@ def make(config: Config): config.BIN_DIR.mkdir(parents=True) all_app_objects = [config.OBJECT_DIR / Path(app_path).parent / (Path(app_path).stem + '.o') for app_path in config.APPS] - for app_path, app_object_path in zip(config.APPS, all_app_objects): - if 'IGNORE_APPS' in config.__dict__ and app_path in config.IGNORE_APPS: - continue - bin_path = config.BIN_DIR / app_path - if not bin_path.parent.exists(): - bin_path.parent.mkdir(parents=True) - object_files = [str(object_path) for _, object_path in compile_list - if object_path not in all_app_objects] - object_files.append(str(app_object_path)) - cmd = ' '.join([config.CC, config.COMMON_FLAGS, *object_files, '-o', str(bin_path), config.LINK_FLAGS]) - complete = subprocess.run(cmd, check=False, shell=True) - if complete.returncode != 0: - error_path = app_path - if error_path: + if arguments.j > 1: # Multi-process + jobs: list[tuple[Path, subprocess.Popen]] = [] + for app_path, app_object_path in zip(config.APPS, all_app_objects): + if 'IGNORE_APPS' in config.__dict__ and app_path in config.IGNORE_APPS: + continue + bin_path = config.BIN_DIR / app_path + if not bin_path.parent.exists(): + bin_path.parent.mkdir(parents=True) + object_files = [str(object_path) for _, object_path in compile_list + if object_path not in all_app_objects] + object_files.append(str(app_object_path)) + cmd = ' '.join([config.CC, config.COMMON_FLAGS, *object_files, '-o', str(bin_path), config.LINK_FLAGS]) + print(ConsoleColor.BLUE + cmd + ConsoleColor.ENDCOLOR) + jobs.insert(0, (app_path, subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, + shell=True))) # FIFO style (will be poped) + if len(jobs) >= arguments.j: # If jobs count is maxed we wait for the oldest one to finished + app_path, oldest_job = jobs.pop() + oldest_job.wait() + if oldest_job.returncode != 0: + error_paths.append((source_path, oldest_job.stdout.read() + oldest_job.stderr.read())) + break + for job_path, job in jobs: # Wait the last jobs to finish + job.wait() + if job.returncode != 0: + error_paths.append((source_path, job.stdout.read() + job.stderr.read())) + else: # Single-process + for app_path, app_object_path in zip(config.APPS, all_app_objects): + if 'IGNORE_APPS' in config.__dict__ and app_path in config.IGNORE_APPS: + continue + bin_path = config.BIN_DIR / app_path + if not bin_path.parent.exists(): + bin_path.parent.mkdir(parents=True) + object_files = [str(object_path) for _, object_path in compile_list + if object_path not in all_app_objects] + object_files.append(str(app_object_path)) + cmd = ' '.join([config.CC, config.COMMON_FLAGS, *object_files, '-o', str(bin_path), config.LINK_FLAGS]) + complete = subprocess.run(cmd, check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True, shell=True) + if complete.returncode != 0: + error_paths.append(app_path) + if error_paths: + for error_path, error_text in error_paths: + print(ConsoleColor.RED + f'Error linking {error_path}:\n' + ConsoleColor.ENDCOLOR + error_text) with open(os.path.join(config.OBJECT_DIR, 'hash.json'), 'w') as hash_file: hash_file.write(json.dumps(hash_dict, indent=1)) - print(f'Error linking {error_path}') sys.exit(1) # Updating header hashes only if everything compiled and linked correctly