Add watch option
This commit is contained in:
Corentin
parent
f556df8c13
commit
64c31f6211
5 changed files with 358 additions and 153 deletions
343
umake.py
343
umake.py
|
|
@ -15,6 +15,7 @@ class Config:
|
|||
APPS = ['app_name'] # Output binaries (need to be found as .cpp directly in SOURCE_DIR)
|
||||
IGNORE_APPS = []
|
||||
JOB_COUNT = int(os.cpu_count() * 0.8) # Concurent jobs (multi-processing)
|
||||
WATCH = False # Watch source modification for auto-compiling
|
||||
|
||||
BIN_DIR = Path('bin') # Output directory (binaries)
|
||||
INCLUDE_DIR = Path('include') # Include directory (header files)
|
||||
|
|
@ -82,162 +83,214 @@ def make(config: Config):
|
|||
config.COMMON_FLAGS += ' ' + config.COMMON_RELEASE_FLAGS
|
||||
config.OBJECT_DIR = config.OBJECT_DIR / 'release'
|
||||
|
||||
# Create list of source to process (tuple[source_path, object_path])
|
||||
compile_list = [(Path(source_file),
|
||||
config.OBJECT_DIR / source_file.parent.relative_to(config.SOURCE_DIR) / (source_file.stem + '.o'))
|
||||
for source_file in config.CPP_SOURCES]
|
||||
todo_list: list[tuple[Path, str]] = []
|
||||
error_paths: list[tuple[Path, str]] = []
|
||||
# Update job count
|
||||
config.JOB_COUNT = arguments.j
|
||||
|
||||
hash_dict = {}
|
||||
if (config.OBJECT_DIR / 'hash.json').exists():
|
||||
with open(config.OBJECT_DIR / 'hash.json', 'r', encoding='utf-8') as hash_file:
|
||||
hash_dict = json.loads(hash_file.read())
|
||||
Builder(config).make()
|
||||
|
||||
# Get source dependencies
|
||||
dependency_dict = {}
|
||||
for source_path, object_path in compile_list:
|
||||
cmd = ' '.join([config.CC, config.COMMON_FLAGS, config.COMPILE_FLAGS, str(source_path), '-M'])
|
||||
job = subprocess.run(cmd, check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
||||
universal_newlines=True, shell=True)
|
||||
if job.returncode != 0:
|
||||
error_paths.append((source_path, job.stdout + job.stderr))
|
||||
break
|
||||
dependency_dict[str(source_path)] = [
|
||||
line for line in job.stdout.split('.o: ')[1].replace('\n', '').replace('\\ ', '').split(' ') if line]
|
||||
if error_paths:
|
||||
for error_path, error_text in error_paths:
|
||||
print(f'{ConsoleColor.RED}Error checking dependencies for {error_path}:'
|
||||
f'\n{ConsoleColor.ENDCOLOR + error_text}')
|
||||
with open(config.OBJECT_DIR / 'hash.json', 'w', encoding='utf-8') as hash_file:
|
||||
hash_file.write(json.dumps(hash_dict, indent=1))
|
||||
sys.exit(1)
|
||||
|
||||
# Check source file and generate compilation commands to execute
|
||||
for source_path, object_path in compile_list:
|
||||
cmd = ' '.join([config.CC, config.COMMON_FLAGS, config.COMPILE_FLAGS,
|
||||
str(source_path), '-c', '-o', str(object_path)])
|
||||
if not object_path.parent.exists():
|
||||
object_path.parent.mkdir(parents=True)
|
||||
source_hash = get_hash(source_path)
|
||||
dependency_changed = False
|
||||
for dependency_path in dependency_dict[str(source_path)]:
|
||||
dependency_hash = get_hash(dependency_path)
|
||||
if str(dependency_path) not in hash_dict or hash_dict[str(dependency_path)] != dependency_hash:
|
||||
dependency_changed = True
|
||||
print(f'{ConsoleColor.ORANGE}Dependency changed for {source_path} :'
|
||||
f' {dependency_path} {ConsoleColor.ENDCOLOR}')
|
||||
break
|
||||
if (dependency_changed or not object_path.exists()
|
||||
or str(source_path) not in hash_dict or hash_dict[str(source_path)] != source_hash):
|
||||
todo_list.append((source_path, cmd))
|
||||
continue
|
||||
class Builder:
|
||||
def __init__(self, config: Config):
|
||||
self._config = config
|
||||
|
||||
if not todo_list and all(
|
||||
[(config.BIN_DIR / app_path).exists() for app_path in config.APPS if app_path not in config.IGNORE_APPS]):
|
||||
print(f'{ConsoleColor.GREEN}Nothing to do{ConsoleColor.ENDCOLOR}')
|
||||
return
|
||||
# Create list of source to process (tuple[source_path, object_path])
|
||||
self._compile_dict: dict[Path, Path] = {
|
||||
Path(source_file): (self._config.OBJECT_DIR / source_file.parent.relative_to(self._config.SOURCE_DIR)
|
||||
/ (source_file.stem + '.o'))
|
||||
for source_file in self._config.CPP_SOURCES}
|
||||
self._todo_dict: dict[Path, str] = {}
|
||||
|
||||
# Running compilation processes
|
||||
if not os.path.exists(config.OBJECT_DIR):
|
||||
os.makedirs(config.OBJECT_DIR)
|
||||
if arguments.j > 1: # Multi-process
|
||||
jobs: list[tuple[Path, subprocess.Popen]] = []
|
||||
for source_path, cmd in todo_list:
|
||||
print(ConsoleColor.BLUE + cmd + ConsoleColor.ENDCOLOR)
|
||||
jobs.insert(0, (source_path, subprocess.Popen(
|
||||
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True,
|
||||
shell=True))) # FIFO style (will be poped)
|
||||
if len(jobs) >= arguments.j: # If jobs count is maxed we wait for the oldest one to finished
|
||||
job_path, oldest_job = jobs.pop()
|
||||
oldest_job.wait()
|
||||
if oldest_job.returncode != 0:
|
||||
error_paths.append((job_path, oldest_job.stdout.read() + oldest_job.stderr.read()))
|
||||
break
|
||||
# Update hash if no error
|
||||
for dependency_path in dependency_dict[str(job_path)]:
|
||||
hash_dict[str(dependency_path)] = get_hash(dependency_path)
|
||||
del source_path
|
||||
for job_path, job in jobs: # Wait the last jobs to finish
|
||||
job.wait()
|
||||
if job.returncode != 0:
|
||||
error_paths.append((job_path, job.stdout.read() + job.stderr.read()))
|
||||
else:
|
||||
# Update hash if no error
|
||||
for dependency_path in dependency_dict[str(job_path)]:
|
||||
hash_dict[str(dependency_path)] = get_hash(dependency_path)
|
||||
else: # Single-process
|
||||
for source_path, cmd in todo_list:
|
||||
print(ConsoleColor.BLUE + cmd + ConsoleColor.ENDCOLOR)
|
||||
self._hash_dict: dict[str, str] = {}
|
||||
if (self._config.OBJECT_DIR / 'hash.json').exists():
|
||||
with open(self._config.OBJECT_DIR / 'hash.json', 'r', encoding='utf-8') as hash_file:
|
||||
self._hash_dict = json.loads(hash_file.read())
|
||||
|
||||
# Get source dependencies
|
||||
error_paths: list[tuple[Path, str]] = []
|
||||
self._dependency_dict: dict[str, list[str]] = {}
|
||||
for source_path, _object_path in self._compile_dict.items():
|
||||
cmd = ' '.join(
|
||||
[self._config.CC, self._config.COMMON_FLAGS, self._config.COMPILE_FLAGS, str(source_path), '-M'])
|
||||
job = subprocess.run(cmd, check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
||||
universal_newlines=True, shell=True)
|
||||
if job.returncode != 0:
|
||||
error_paths.append((source_path, job.stdout + job.stderr))
|
||||
break
|
||||
# Update hash if no error
|
||||
for dependency_path in dependency_dict[str(source_path)]:
|
||||
hash_dict[str(dependency_path)] = get_hash(dependency_path)
|
||||
del source_path
|
||||
if error_paths:
|
||||
for error_path, error_text in error_paths:
|
||||
print(ConsoleColor.RED + f'Error compiling {error_path}:\n' + ConsoleColor.ENDCOLOR + error_text)
|
||||
with open(config.OBJECT_DIR / 'hash.json', 'w', encoding='utf-8') as hash_file:
|
||||
hash_file.write(json.dumps(hash_dict, indent=1))
|
||||
sys.exit(1)
|
||||
self._dependency_dict[str(source_path)] = [
|
||||
line for line in job.stdout.split('.o: ')[1].replace('\n', '').replace('\\ ', '').split(' ') if line]
|
||||
if error_paths:
|
||||
for error_path, error_text in error_paths:
|
||||
print(f'{ConsoleColor.RED}Error checking dependencies for {error_path}:'
|
||||
f'\n{ConsoleColor.ENDCOLOR + error_text}')
|
||||
with open(self._config.OBJECT_DIR / 'hash.json', 'w', encoding='utf-8') as hash_file:
|
||||
hash_file.write(json.dumps(self._hash_dict, indent=1))
|
||||
sys.exit(1)
|
||||
|
||||
# Running linking processes
|
||||
if not config.BIN_DIR.exists():
|
||||
config.BIN_DIR.mkdir(parents=True)
|
||||
all_app_objects = [config.OBJECT_DIR / Path(app_path).parent / (Path(app_path).stem + '.o')
|
||||
for app_path in config.APPS]
|
||||
if arguments.j > 1: # Multi-process
|
||||
jobs: list[tuple[Path, subprocess.Popen]] = []
|
||||
for app_path, app_object_path in zip(config.APPS, all_app_objects):
|
||||
if app_path in config.IGNORE_APPS:
|
||||
continue
|
||||
bin_path = config.BIN_DIR / app_path
|
||||
if not bin_path.parent.exists():
|
||||
bin_path.parent.mkdir(parents=True)
|
||||
object_files = [str(object_path) for _, object_path in compile_list
|
||||
if object_path not in all_app_objects]
|
||||
object_files.append(str(app_object_path))
|
||||
cmd = ' '.join([config.CC, config.COMMON_FLAGS, *object_files, '-o', str(bin_path), config.LINK_FLAGS])
|
||||
print(ConsoleColor.BLUE + cmd + ConsoleColor.ENDCOLOR)
|
||||
jobs.insert(0, (app_path, subprocess.Popen(
|
||||
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True,
|
||||
shell=True))) # FIFO style (will be poped)
|
||||
if len(jobs) >= arguments.j: # If jobs count is maxed we wait for the oldest one to finished
|
||||
app_path, oldest_job = jobs.pop()
|
||||
oldest_job.wait()
|
||||
if oldest_job.returncode != 0:
|
||||
error_paths.append((app_path, oldest_job.stdout.read() + oldest_job.stderr.read()))
|
||||
def make(self):
|
||||
self._populate_todo_dict()
|
||||
self._compile_sources()
|
||||
|
||||
if self._config.WATCH:
|
||||
from watch import Watcher, WatchFlag # pylint: disable=import-outside-toplevel
|
||||
|
||||
def callback(path: Path, _flags: WatchFlag):
|
||||
source_hash = get_hash(path)
|
||||
if source_hash != self._hash_dict[str(path)]:
|
||||
print(f'{ConsoleColor.ORANGE}Source file changed : {path} {ConsoleColor.ENDCOLOR}')
|
||||
self._populate_todo_dict()
|
||||
self._compile_sources()
|
||||
|
||||
watcher = Watcher()
|
||||
for source_path in self._compile_dict:
|
||||
watcher.register(source_path, WatchFlag.MODIFY, callback)
|
||||
try:
|
||||
watcher.watch()
|
||||
except KeyboardInterrupt:
|
||||
print('\rExit')
|
||||
|
||||
def _populate_todo_dict(self):
|
||||
"""Check source file and generate compilation commands to execute."""
|
||||
for source_path, object_path in self._compile_dict.items():
|
||||
if not object_path.parent.exists():
|
||||
object_path.parent.mkdir(parents=True)
|
||||
source_hash = get_hash(source_path)
|
||||
dependency_changed = False
|
||||
for path in self._dependency_dict[str(source_path)]:
|
||||
dependency_hash = get_hash(path)
|
||||
if str(path) not in self._hash_dict or self._hash_dict[str(path)] != dependency_hash:
|
||||
dependency_changed = True
|
||||
print(f'{ConsoleColor.ORANGE}Dependency changed for {source_path} : {path} {ConsoleColor.ENDCOLOR}')
|
||||
break
|
||||
for job_path, job in jobs: # Wait the last jobs to finish
|
||||
job.wait()
|
||||
if job.returncode != 0:
|
||||
error_paths.append((job_path, job.stdout.read() + job.stderr.read()))
|
||||
else: # Single-process
|
||||
for app_path, app_object_path in zip(config.APPS, all_app_objects):
|
||||
if app_path in config.IGNORE_APPS:
|
||||
if (dependency_changed or not object_path.exists()
|
||||
or str(source_path) not in self._hash_dict or self._hash_dict[str(source_path)] != source_hash):
|
||||
self._todo_dict[source_path] = self._generate_compilation_command(source_path, object_path)
|
||||
continue
|
||||
bin_path = config.BIN_DIR / app_path
|
||||
if not bin_path.parent.exists():
|
||||
bin_path.parent.mkdir(parents=True)
|
||||
object_files = [str(object_path) for _, object_path in compile_list
|
||||
if object_path not in all_app_objects]
|
||||
object_files.append(str(app_object_path))
|
||||
cmd = ' '.join([config.CC, config.COMMON_FLAGS, *object_files, '-o', str(bin_path), config.LINK_FLAGS])
|
||||
print(ConsoleColor.BLUE + cmd + ConsoleColor.ENDCOLOR)
|
||||
job = subprocess.run(cmd, check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
||||
universal_newlines=True, shell=True)
|
||||
if job.returncode != 0:
|
||||
error_paths.append((app_path, job.stdout + job.stderr))
|
||||
if error_paths:
|
||||
for error_path, error_text in error_paths:
|
||||
print(ConsoleColor.RED + f'Error linking {error_path}:\n' + ConsoleColor.ENDCOLOR + error_text)
|
||||
with open(os.path.join(config.OBJECT_DIR, 'hash.json'), 'w', encoding='utf-8') as hash_file:
|
||||
hash_file.write(json.dumps(hash_dict, indent=1))
|
||||
sys.exit(1)
|
||||
|
||||
with open(config.OBJECT_DIR / 'hash.json', 'w', encoding='utf-8') as hash_file:
|
||||
hash_file.write(json.dumps(hash_dict, indent=1))
|
||||
def _generate_compilation_command(self, source_path: Path, object_path: Path) -> str:
|
||||
return ' '.join([self._config.CC, self._config.COMMON_FLAGS, self._config.COMPILE_FLAGS,
|
||||
str(source_path), '-c', '-o', str(object_path)])
|
||||
|
||||
def _compile_sources(self):
|
||||
if not self._todo_dict and all(
|
||||
(self._config.BIN_DIR / app_path).exists()
|
||||
for app_path in self._config.APPS if app_path not in self._config.IGNORE_APPS):
|
||||
print(f'{ConsoleColor.GREEN}Nothing to do{ConsoleColor.ENDCOLOR}')
|
||||
return
|
||||
|
||||
# Running compilation processes
|
||||
if not os.path.exists(self._config.OBJECT_DIR):
|
||||
os.makedirs(self._config.OBJECT_DIR)
|
||||
error_paths: list[tuple[Path, str]] = []
|
||||
if self._config.JOB_COUNT > 1: # Multi-process
|
||||
jobs: list[tuple[Path, subprocess.Popen]] = []
|
||||
completed_paths: list[Path] = []
|
||||
for source_path, cmd in self._todo_dict.items():
|
||||
print(ConsoleColor.BLUE + cmd + ConsoleColor.ENDCOLOR)
|
||||
jobs.insert(0, (source_path, subprocess.Popen(
|
||||
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True,
|
||||
shell=True))) # FIFO style (will be poped)
|
||||
if len(jobs) >= self._config.JOB_COUNT: # If jobs count is maxed we wait for the oldest one to finished
|
||||
job_path, oldest_job = jobs.pop()
|
||||
oldest_job.wait()
|
||||
if oldest_job.returncode != 0:
|
||||
error_paths.append((job_path, oldest_job.stdout.read() + oldest_job.stderr.read()))
|
||||
break
|
||||
# Update hash if no error
|
||||
for dependency_path in self._dependency_dict[str(job_path)]:
|
||||
self._hash_dict[str(dependency_path)] = get_hash(dependency_path)
|
||||
completed_paths.append(source_path)
|
||||
for source_path in completed_paths:
|
||||
del self._todo_dict[source_path]
|
||||
|
||||
for job_path, job in jobs: # Wait the last jobs to finish
|
||||
job.wait()
|
||||
if job.returncode != 0:
|
||||
error_paths.append((job_path, job.stdout.read() + job.stderr.read()))
|
||||
else:
|
||||
# Update hash if no error
|
||||
for dependency_path in self._dependency_dict[str(job_path)]:
|
||||
self._hash_dict[str(dependency_path)] = get_hash(dependency_path)
|
||||
del self._todo_dict[job_path]
|
||||
else: # Single-process
|
||||
completed_paths: list[Path] = []
|
||||
for source_path, cmd in self._todo_dict.items():
|
||||
print(ConsoleColor.BLUE + cmd + ConsoleColor.ENDCOLOR)
|
||||
job = subprocess.run(cmd, check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
||||
universal_newlines=True, shell=True)
|
||||
if job.returncode != 0:
|
||||
error_paths.append((source_path, job.stdout + job.stderr))
|
||||
break
|
||||
# Update hash if no error
|
||||
for dependency_path in self._dependency_dict[str(source_path)]:
|
||||
self._hash_dict[str(dependency_path)] = get_hash(dependency_path)
|
||||
completed_paths.append(source_path)
|
||||
for source_path in completed_paths:
|
||||
del self._todo_dict[source_path]
|
||||
|
||||
if error_paths:
|
||||
for error_path, error_text in error_paths:
|
||||
print(ConsoleColor.RED + f'Error compiling {error_path}:\n' + ConsoleColor.ENDCOLOR + error_text)
|
||||
with open(self._config.OBJECT_DIR / 'hash.json', 'w', encoding='utf-8') as hash_file:
|
||||
hash_file.write(json.dumps(self._hash_dict, indent=1))
|
||||
sys.exit(1)
|
||||
|
||||
# Running linking processes
|
||||
if not self._config.BIN_DIR.exists():
|
||||
self._config.BIN_DIR.mkdir(parents=True)
|
||||
all_app_objects = [self._config.OBJECT_DIR / Path(app_path).parent / (Path(app_path).stem + '.o')
|
||||
for app_path in self._config.APPS]
|
||||
if self._config.JOB_COUNT > 1: # Multi-process
|
||||
jobs: list[tuple[Path, subprocess.Popen]] = []
|
||||
for app_path, app_object_path in zip(self._config.APPS, all_app_objects):
|
||||
if app_path in self._config.IGNORE_APPS:
|
||||
continue
|
||||
bin_path = self._config.BIN_DIR / app_path
|
||||
if not bin_path.parent.exists():
|
||||
bin_path.parent.mkdir(parents=True)
|
||||
object_files = [str(object_path) for object_path in self._compile_dict.values()
|
||||
if object_path not in all_app_objects]
|
||||
object_files.append(str(app_object_path))
|
||||
cmd = ' '.join([self._config.CC, self._config.COMMON_FLAGS, *object_files, '-o', str(bin_path),
|
||||
self._config.LINK_FLAGS])
|
||||
print(ConsoleColor.BLUE + cmd + ConsoleColor.ENDCOLOR)
|
||||
jobs.insert(0, (app_path, subprocess.Popen(
|
||||
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True,
|
||||
shell=True))) # FIFO style (will be poped)
|
||||
if len(jobs) >= self._config.JOB_COUNT: # If jobs count is maxed we wait for the oldest one to finished
|
||||
app_path, oldest_job = jobs.pop()
|
||||
oldest_job.wait()
|
||||
if oldest_job.returncode != 0:
|
||||
error_paths.append((app_path, oldest_job.stdout.read() + oldest_job.stderr.read()))
|
||||
break
|
||||
for job_path, job in jobs: # Wait the last jobs to finish
|
||||
job.wait()
|
||||
if job.returncode != 0:
|
||||
error_paths.append((job_path, job.stdout.read() + job.stderr.read()))
|
||||
else: # Single-process
|
||||
for app_path, app_object_path in zip(self._config.APPS, all_app_objects):
|
||||
if app_path in self._config.IGNORE_APPS:
|
||||
continue
|
||||
bin_path = self._config.BIN_DIR / app_path
|
||||
if not bin_path.parent.exists():
|
||||
bin_path.parent.mkdir(parents=True)
|
||||
object_files = [str(object_path) for object_path in self._compile_dict.values()
|
||||
if object_path not in all_app_objects]
|
||||
object_files.append(str(app_object_path))
|
||||
cmd = ' '.join([self._config.CC, self._config.COMMON_FLAGS, *object_files, '-o', str(bin_path),
|
||||
self._config.LINK_FLAGS])
|
||||
print(ConsoleColor.BLUE + cmd + ConsoleColor.ENDCOLOR)
|
||||
job = subprocess.run(cmd, check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
||||
universal_newlines=True, shell=True)
|
||||
if job.returncode != 0:
|
||||
error_paths.append((app_path, job.stdout + job.stderr))
|
||||
if error_paths:
|
||||
for error_path, error_text in error_paths:
|
||||
print(ConsoleColor.RED + f'Error linking {error_path}:\n' + ConsoleColor.ENDCOLOR + error_text)
|
||||
with open(os.path.join(self._config.OBJECT_DIR, 'hash.json'), 'w', encoding='utf-8') as hash_file:
|
||||
hash_file.write(json.dumps(self._hash_dict, indent=1))
|
||||
sys.exit(1)
|
||||
|
||||
with open(self._config.OBJECT_DIR / 'hash.json', 'w', encoding='utf-8') as hash_file:
|
||||
hash_file.write(json.dumps(self._hash_dict, indent=1))
|
||||
|
||||
print(f'{ConsoleColor.GREEN}Compilation done{ConsoleColor.ENDCOLOR}')
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue