From bde16591c5b72d35118d1c7a93c404e0ba2b1bb6 Mon Sep 17 00:00:00 2001 From: Corentin Date: Tue, 29 Jun 2021 02:25:32 +0900 Subject: [PATCH] Dependecy checks + minor fixes --- umake.py | 85 ++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 55 insertions(+), 30 deletions(-) diff --git a/umake.py b/umake.py index efe19de..56e6050 100755 --- a/umake.py +++ b/umake.py @@ -74,21 +74,31 @@ def make(config: Config): compile_list = [(Path(source_file), config.OBJECT_DIR / source_file.parent.relative_to(config.SOURCE_DIR) / (source_file.stem + '.o')) for source_file in config.CPP_SOURCES] - todo_list = [] + todo_list: list[tuple[Path, str]] = [] + error_paths: list[tuple[Path, str]] = [] hash_dict = {} if (config.OBJECT_DIR / 'hash.json').exists(): with open(config.OBJECT_DIR / 'hash.json', 'r') as hash_file: hash_dict = json.loads(hash_file.read()) - # Check header files (if any modification is done all source file will be processed to avoid any problem) - header_changed = False - header_hash_dict = {} - for header_path in config.CPP_HEADERS: - header_hash = get_hash(header_path) - if str(header_path) not in hash_dict or hash_dict[str(header_path)] != header_hash: - header_hash_dict[str(header_path)] = header_hash - header_changed = True + # Get source dependencies + dependency_dict = {} + for source_path, object_path in compile_list: + cmd = ' '.join([config.CC, config.COMMON_FLAGS, config.COMPILE_FLAGS, str(source_path), '-M']) + job = subprocess.run(cmd, check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True, shell=True) + if job.returncode != 0: + error_paths.append((source_path, job.stdout.read() + job.stderr.read())) + break + dependency_dict[str(source_path)] = job.stdout.split('.o: ')[1].replace('\n', '').replace('\\ ', '').split(' ') + if error_paths: + for error_path, error_text in error_paths: + print(ConsoleColor.RED + f'Error checking dependencies for {error_path}:\n' + + ConsoleColor.ENDCOLOR + error_text) + with open(config.OBJECT_DIR / 'hash.json', 'w') as hash_file: + hash_file.write(json.dumps(hash_dict, indent=1)) + sys.exit(1) # Check source file and generate compilation commands to execute for source_path, object_path in compile_list: @@ -97,44 +107,61 @@ def make(config: Config): if not object_path.parent.exists(): object_path.parent.mkdir(parents=True) source_hash = get_hash(source_path) - if (header_changed or not object_path.exists() + dependency_changed = False + for dependency_path in dependency_dict[str(source_path)]: + dependency_hash = get_hash(dependency_path) + if str(dependency_path) not in hash_dict or hash_dict[str(dependency_path)] != dependency_hash: + dependency_changed = True + print(ConsoleColor.ORANGE + f'Dependency changed for {source_path} : {dependency_path}' + + ConsoleColor.ENDCOLOR) + break + if (dependency_changed or not object_path.exists() or str(source_path) not in hash_dict or hash_dict[str(source_path)] != source_hash): - todo_list.append((source_path, source_hash, cmd)) + todo_list.append((source_path, cmd)) continue + if not todo_list: + print(ConsoleColor.GREEN + 'Nothing to do' + ConsoleColor.ENDCOLOR) + return + # Running compilation processes if not os.path.exists(config.OBJECT_DIR): os.makedirs(config.OBJECT_DIR) - error_paths: list[tuple[Path, str]] = [] if arguments.j > 1: # Multi-process - jobs: list[tuple[Path, Path, subprocess.Popen]] = [] - for source_path, source_hash, cmd in todo_list: + jobs: list[tuple[Path, subprocess.Popen]] = [] + for source_path, cmd in todo_list: print(ConsoleColor.BLUE + cmd + ConsoleColor.ENDCOLOR) - jobs.insert(0, (source_path, source_hash, subprocess.Popen( + jobs.insert(0, (source_path, subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, shell=True))) # FIFO style (will be poped) if len(jobs) >= arguments.j: # If jobs count is maxed we wait for the oldest one to finished - job_path, job_hash, oldest_job = jobs.pop() + job_path, oldest_job = jobs.pop() oldest_job.wait() if oldest_job.returncode != 0: error_paths.append((job_path, oldest_job.stdout.read() + oldest_job.stderr.read())) break - hash_dict[str(job_path)] = job_hash # Update hash if no error - for job_path, job_hash, job in jobs: # Wait the last jobs to finish + # Update hash if no error + for dependency_path in dependency_dict[str(job_path)]: + hash_dict[str(dependency_path)] = get_hash(dependency_path) + for job_path, job in jobs: # Wait the last jobs to finish job.wait() if job.returncode != 0: error_paths.append((job_path, job.stdout.read() + job.stderr.read())) else: - hash_dict[str(job_path)] = job_hash # Update hash if no error + # Update hash if no error + for dependency_path in dependency_dict[str(job_path)]: + hash_dict[str(dependency_path)] = get_hash(dependency_path) else: # Single-process for source_path, source_hash, cmd in todo_list: - print(cmd) - job = subprocess.run(cmd, check=False, stdin=subprocess.PIPE, stderr=subprocess.PIPE, + print(ConsoleColor.BLUE + cmd + ConsoleColor.ENDCOLOR) + job = subprocess.run(cmd, check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, shell=True) if job.returncode != 0: - error_paths.append((source_path, job.stdout.read() + job.stderr.read())) + error_paths.append((source_path, job.stdout + job.stderr)) break - hash_dict[str(source_path)] = source_hash # Update hash if no error + # Update hash if no error + for dependency_path in dependency_dict[str(source_path)]: + hash_dict[str(dependency_path)] = get_hash(dependency_path) if error_paths: for error_path, error_text in error_paths: print(ConsoleColor.RED + f'Error compiling {error_path}:\n' + ConsoleColor.ENDCOLOR + error_text) @@ -184,10 +211,11 @@ def make(config: Config): if object_path not in all_app_objects] object_files.append(str(app_object_path)) cmd = ' '.join([config.CC, config.COMMON_FLAGS, *object_files, '-o', str(bin_path), config.LINK_FLAGS]) - complete = subprocess.run(cmd, check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True, shell=True) - if complete.returncode != 0: - error_paths.append(app_path) + print(ConsoleColor.BLUE + cmd + ConsoleColor.ENDCOLOR) + job = subprocess.run(cmd, check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True, shell=True) + if job.returncode != 0: + error_paths.append((source_path, job.stdout + job.stderr)) if error_paths: for error_path, error_text in error_paths: print(ConsoleColor.RED + f'Error linking {error_path}:\n' + ConsoleColor.ENDCOLOR + error_text) @@ -195,8 +223,5 @@ def make(config: Config): hash_file.write(json.dumps(hash_dict, indent=1)) sys.exit(1) - # Updating header hashes only if everything compiled and linked correctly - for header_path in header_hash_dict: - hash_dict[header_path] = header_hash_dict[header_path] with open(config.OBJECT_DIR / 'hash.json', 'w') as hash_file: hash_file.write(json.dumps(hash_dict, indent=1))