Multiprocess linking + better messages
This commit is contained in:
Corentin
parent
63574f95cf
commit
2abe6c7953
1 changed files with 75 additions and 32 deletions
107
umake.py
107
umake.py
|
|
@ -8,7 +8,6 @@ from pathlib import Path
|
|||
import subprocess
|
||||
import shutil
|
||||
import sys
|
||||
from typing import List
|
||||
|
||||
|
||||
class Config:
|
||||
|
|
@ -31,6 +30,18 @@ class Config:
|
|||
CPP_SOURCES = SOURCE_DIR.rglob('*.cpp')
|
||||
|
||||
|
||||
class ConsoleColor:
|
||||
"""Simple shortcut to use colors in console"""
|
||||
HEADER = '\033[95m'
|
||||
BLUE = '\033[94m'
|
||||
GREEN = '\033[92m'
|
||||
ORANGE = '\033[93m'
|
||||
RED = '\033[91m'
|
||||
ENDCOLOR = '\033[0m'
|
||||
BOLD = '\033[1m'
|
||||
UNDERLINE = '\033[4m'
|
||||
|
||||
|
||||
def get_hash(path: Path) -> str:
|
||||
with open(path, 'r') as hashing_file:
|
||||
hash_obj = hashlib.md5()
|
||||
|
|
@ -94,36 +105,39 @@ def make(config: Config):
|
|||
# Running compilation processes
|
||||
if not os.path.exists(config.OBJECT_DIR):
|
||||
os.makedirs(config.OBJECT_DIR)
|
||||
error_path = None
|
||||
error_paths: list[tuple[Path, str]] = []
|
||||
if arguments.j > 1: # Multi-process
|
||||
jobs: List[subprocess.Popen] = []
|
||||
jobs: list[tuple[Path, Path, subprocess.Popen]] = []
|
||||
for source_path, source_hash, cmd in todo_list:
|
||||
print(cmd)
|
||||
jobs.insert(0, (source_path, source_hash, subprocess.Popen(cmd, shell=True))) # FIFO style (will be poped)
|
||||
print(ConsoleColor.BLUE + cmd + ConsoleColor.ENDCOLOR)
|
||||
jobs.insert(0, (source_path, source_hash, subprocess.Popen(
|
||||
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True,
|
||||
shell=True))) # FIFO style (will be poped)
|
||||
if len(jobs) >= arguments.j: # If jobs count is maxed we wait for the oldest one to finished
|
||||
job_path, job_hash, oldest_job = jobs.pop()
|
||||
oldest_job.wait()
|
||||
if oldest_job.returncode != 0:
|
||||
error_path = job_path
|
||||
error_paths.append((job_path, oldest_job.stdout.read() + oldest_job.stderr.read()))
|
||||
break
|
||||
hash_dict[str(job_path)] = job_hash # Update hash if no error
|
||||
if error_path is None:
|
||||
for job_path, job_hash, job in jobs: # Wait the last jobs to finish
|
||||
job.wait()
|
||||
if job.returncode != 0:
|
||||
error_path = job_path
|
||||
break
|
||||
for job_path, job_hash, job in jobs: # Wait the last jobs to finish
|
||||
job.wait()
|
||||
if job.returncode != 0:
|
||||
error_paths.append((job_path, job.stdout.read() + job.stderr.read()))
|
||||
else:
|
||||
hash_dict[str(job_path)] = job_hash # Update hash if no error
|
||||
else: # Single-process
|
||||
for source_path, source_hash, cmd in todo_list:
|
||||
print(cmd)
|
||||
complete = subprocess.run(cmd, check=False, shell=True)
|
||||
if complete.returncode != 0:
|
||||
error_path = source_path
|
||||
job = subprocess.run(cmd, check=False, stdin=subprocess.PIPE, stderr=subprocess.PIPE,
|
||||
universal_newlines=True, shell=True)
|
||||
if job.returncode != 0:
|
||||
error_paths.append((source_path, job.stdout.read() + job.stderr.read()))
|
||||
break
|
||||
hash_dict[str(source_path)] = source_hash # Update hash if no error
|
||||
if error_path:
|
||||
print(f'Error compiling {error_path}')
|
||||
if error_paths:
|
||||
for error_path, error_text in error_paths:
|
||||
print(ConsoleColor.RED + f'Error compiling {error_path}:\n' + ConsoleColor.ENDCOLOR + error_text)
|
||||
with open(config.OBJECT_DIR / 'hash.json', 'w') as hash_file:
|
||||
hash_file.write(json.dumps(hash_dict, indent=1))
|
||||
sys.exit(1)
|
||||
|
|
@ -133,23 +147,52 @@ def make(config: Config):
|
|||
config.BIN_DIR.mkdir(parents=True)
|
||||
all_app_objects = [config.OBJECT_DIR / Path(app_path).parent / (Path(app_path).stem + '.o')
|
||||
for app_path in config.APPS]
|
||||
for app_path, app_object_path in zip(config.APPS, all_app_objects):
|
||||
if 'IGNORE_APPS' in config.__dict__ and app_path in config.IGNORE_APPS:
|
||||
continue
|
||||
bin_path = config.BIN_DIR / app_path
|
||||
if not bin_path.parent.exists():
|
||||
bin_path.parent.mkdir(parents=True)
|
||||
object_files = [str(object_path) for _, object_path in compile_list
|
||||
if object_path not in all_app_objects]
|
||||
object_files.append(str(app_object_path))
|
||||
cmd = ' '.join([config.CC, config.COMMON_FLAGS, *object_files, '-o', str(bin_path), config.LINK_FLAGS])
|
||||
complete = subprocess.run(cmd, check=False, shell=True)
|
||||
if complete.returncode != 0:
|
||||
error_path = app_path
|
||||
if error_path:
|
||||
if arguments.j > 1: # Multi-process
|
||||
jobs: list[tuple[Path, subprocess.Popen]] = []
|
||||
for app_path, app_object_path in zip(config.APPS, all_app_objects):
|
||||
if 'IGNORE_APPS' in config.__dict__ and app_path in config.IGNORE_APPS:
|
||||
continue
|
||||
bin_path = config.BIN_DIR / app_path
|
||||
if not bin_path.parent.exists():
|
||||
bin_path.parent.mkdir(parents=True)
|
||||
object_files = [str(object_path) for _, object_path in compile_list
|
||||
if object_path not in all_app_objects]
|
||||
object_files.append(str(app_object_path))
|
||||
cmd = ' '.join([config.CC, config.COMMON_FLAGS, *object_files, '-o', str(bin_path), config.LINK_FLAGS])
|
||||
print(ConsoleColor.BLUE + cmd + ConsoleColor.ENDCOLOR)
|
||||
jobs.insert(0, (app_path, subprocess.Popen(
|
||||
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True,
|
||||
shell=True))) # FIFO style (will be poped)
|
||||
if len(jobs) >= arguments.j: # If jobs count is maxed we wait for the oldest one to finished
|
||||
app_path, oldest_job = jobs.pop()
|
||||
oldest_job.wait()
|
||||
if oldest_job.returncode != 0:
|
||||
error_paths.append((source_path, oldest_job.stdout.read() + oldest_job.stderr.read()))
|
||||
break
|
||||
for job_path, job in jobs: # Wait the last jobs to finish
|
||||
job.wait()
|
||||
if job.returncode != 0:
|
||||
error_paths.append((source_path, job.stdout.read() + job.stderr.read()))
|
||||
else: # Single-process
|
||||
for app_path, app_object_path in zip(config.APPS, all_app_objects):
|
||||
if 'IGNORE_APPS' in config.__dict__ and app_path in config.IGNORE_APPS:
|
||||
continue
|
||||
bin_path = config.BIN_DIR / app_path
|
||||
if not bin_path.parent.exists():
|
||||
bin_path.parent.mkdir(parents=True)
|
||||
object_files = [str(object_path) for _, object_path in compile_list
|
||||
if object_path not in all_app_objects]
|
||||
object_files.append(str(app_object_path))
|
||||
cmd = ' '.join([config.CC, config.COMMON_FLAGS, *object_files, '-o', str(bin_path), config.LINK_FLAGS])
|
||||
complete = subprocess.run(cmd, check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
||||
universal_newlines=True, shell=True)
|
||||
if complete.returncode != 0:
|
||||
error_paths.append(app_path)
|
||||
if error_paths:
|
||||
for error_path, error_text in error_paths:
|
||||
print(ConsoleColor.RED + f'Error linking {error_path}:\n' + ConsoleColor.ENDCOLOR + error_text)
|
||||
with open(os.path.join(config.OBJECT_DIR, 'hash.json'), 'w') as hash_file:
|
||||
hash_file.write(json.dumps(hash_dict, indent=1))
|
||||
print(f'Error linking {error_path}')
|
||||
sys.exit(1)
|
||||
|
||||
# Updating header hashes only if everything compiled and linked correctly
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue