From 1571f570bf886de87a3f7d760c27981567517f3f Mon Sep 17 00:00:00 2001 From: Corentin Date: Fri, 13 Nov 2020 21:00:02 +0900 Subject: [PATCH] Initial commit --- .gitignore | 1 + LICENSE.txt | 29 ++++++++++ README.md | 53 ++++++++++++++++++ make.py | 34 ++++++++++++ makefile | 13 +++++ umake.py | 151 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 281 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE.txt create mode 100644 README.md create mode 100644 make.py create mode 100644 makefile create mode 100755 umake.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0d20b64 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.pyc diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..ca0f3b9 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2020, Corentin Risselin +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..7490a22 --- /dev/null +++ b/README.md @@ -0,0 +1,53 @@ +# µmake + +µmake is a small python script to easily and quickly compile small or medium-sized C/C++ projects. It is very light and designed to be modified to adapt to specific usage. + +The advantages of this tool are : + +* recursive search of all header/source files in their respective directories (like makefile wildcards) + +* multiple binary output made easy (`app1` will be linked from `app1.cpp`, etc, without conflicts) + +* hash system to avoid time check issues (same file content, remote files problems, etc) + + +## Use + +This repository is meant to be used as a submodule of a C/C++ project. + +* Use `git submodule add` to include this tool anywhere in your project. + +* Copy `makefile` and `make.py` templates in your project root folder. + +* Update `UMAKE_PATH` in the copied `makefile` with the path to this submodule. + +* In `make.py` update the `Config` template with the desired flags and options. + +By default 3 targets can be used `make` (or `make all` as usual default of makefiles), `make debug` and `make clean`. + +**Be careful** : `clean` will delete the object and binary output directories defined in `make.py`. + + +## How it works + +### Project layout + +µmake expects all the header and source files to be place in a directories specified in the configuration (in `make.py`). +Object files and final binaries are outputed also in directories specified in the configuration but objects files in `debug` and `release` (default) are separated. + +The separation of objects files is meant to switch mode quickly (only binaries are rebuilt). + +To determine if a source file needs to be recompiled an hash is saved. The hashed are saved in a JSON file at the root of the object files directory (there are in fact 2 seperate files respectively in `debug` oand `release`). + +If any header file changes every source files will be recompiled to avoid any issue (hence the small or medium-size project target for this tool). This could be address in the future if needed. + + +### Caveats + +* Header files dependencies is not implemented (any change triggers all compilation) + +* Knowing python is recommended to understand the tool for any usage above configuration change. + +* Targets management is not yet implemented (building all binaries), this can be easily address knowing python. A generic solution should be possible in the future. + +* Template configuration is meant for C++ files (cpp/hpp) for now. diff --git a/make.py b/make.py new file mode 100644 index 0000000..e641584 --- /dev/null +++ b/make.py @@ -0,0 +1,34 @@ +#! python3 + +import glob +import os + +from umake import make + + +class Config: + CC = 'g++' # Compiler to call + APPS = ['app_name'] # Output binaries (need to be found as .cpp directly in SOURCE_DIR) + JOB_COUNT = int(os.cpu_count() * 0.8) # Concurent jobs (multi-processing) + + BIN_DIR = 'bin' # Output directory (binaries) + INCLUDE_DIR = 'include' # Include directory (header files) + OBJECT_DIR = 'obj' # Temporary directory (object files) + SOURCE_DIR = 'src' # Source directories + + COMMON_FLAGS = '-std=c++17' # Flags used for comiling and linking + COMMON_DEBUG_FLAGS = '-g' # Flags added in debug mode + COMMON_RELEASE_FLAGS = '-O2 -flto' # Flags added in release mode + COMPILE_FLAGS = f'-Wall -I{INCLUDE_DIR}' # Flags added for compiling (recommandation : `pkg-config --cflags`) + LINK_FLAGS = '' # Flags added for linking (recommandation : `pkg-config --libs`) + + CPP_HEADERS = glob.glob(os.path.join(INCLUDE_DIR, '**', '*.hpp'), recursive=True) + CPP_SOURCES = glob.glob(os.path.join(SOURCE_DIR, '**', '*.cpp'), recursive=True) + + +def main(): + make(Config) + + +if __name__ == '__main__': + main() diff --git a/makefile b/makefile new file mode 100644 index 0000000..db2db6a --- /dev/null +++ b/makefile @@ -0,0 +1,13 @@ +MAKE_PATH="." +UMAKE_PATH="umake" + +.PHONY: all clean + +all: + @PYTHONPATH=$(UMAKE_PATH) python $(MAKE_PATH)/make.py + +debug: + @PYTHONPATH=$(UMAKE_PATH) python $(MAKE_PATH)/make.py --type debug + +clean: + @PYTHONPATH=$(UMAKE_PATH) python $(MAKE_PATH)/make.py --clean diff --git a/umake.py b/umake.py new file mode 100755 index 0000000..5f7db0c --- /dev/null +++ b/umake.py @@ -0,0 +1,151 @@ +#! python3 + +from argparse import ArgumentParser +import hashlib +import json +import os +import subprocess +import shutil +import sys +from typing import List + + +class Config: + CC = 'g++' + APPS = ['app_name'] + JOB_COUNT = 1 + + BIN_DIR = 'bin' + INCLUDE_DIR = 'include' + OBJECT_DIR = 'obj' + SOURCE_DIR = 'src' + + COMMON_FLAGS = '-std=c++17' + COMMON_DEBUG_FLAGS = '-g' + COMMON_RELEASE_FLAGS = '-O2 -flto' + COMPILE_FLAGS = f'-Wall -I{INCLUDE_DIR}' + LINK_FLAGS = '' + + CPP_HEADERS = [] + CPP_SOURCES = [] + + +def get_hash(path: str) -> str: + with open(path, 'r') as hashing_file: + hash_obj = hashlib.md5() + hash_obj.update(hashing_file.read().encode()) + return hash_obj.hexdigest() + + +def make(config: Config): + parser = ArgumentParser() + parser.add_argument('-j', type=int, default=config.JOB_COUNT, help='Jobs count (multi-processing)') + parser.add_argument('--type', default='production', help='Compilation type (release, debug). Default=release') + parser.add_argument('--clean', action='store_true', help='Clean all file instead of building') + arguments = parser.parse_args() + + # Clean action + if arguments.clean: + shutil.rmtree(config.OBJECT_DIR, ignore_errors=True) + shutil.rmtree(config.BIN_DIR, ignore_errors=True) + return + + # Update flags and directories for mode debug/release + if arguments.type == 'debug': + config.COMMON_FLAGS += ' ' + config.COMMON_DEBUG_FLAGS + config.OBJECT_DIR = os.path.join(config.OBJECT_DIR, 'debug') + else: + config.COMMON_FLAGS += ' ' + config.COMMON_RELEASE_FLAGS + config.OBJECT_DIR = os.path.join(config.OBJECT_DIR, 'release') + + # Create list of source to process (tuple[source_path, object_path]) + compile_list = [(source_file, source_file[:-4].replace(config.SOURCE_DIR, config.OBJECT_DIR) + '.o') + for source_file in config.CPP_SOURCES] + todo_list = [] + + hash_dict = {} + if os.path.exists(os.path.join(config.OBJECT_DIR, 'hash.json')): + with open(os.path.join(config.OBJECT_DIR, 'hash.json'), 'r') as hash_file: + hash_dict = json.loads(hash_file.read()) + + # Check header files (if any modification is done all source file will be processed to avoid any problem) + header_changed = False + header_hash_dict = {} + for header_path in config.CPP_HEADERS: + header_hash = get_hash(header_path) + if header_path not in hash_dict or hash_dict[header_path] != header_hash: + header_hash_dict[header_path] = header_hash + header_changed = True + + # Check source file and generate compilation commands to execute + for source_path, object_path in compile_list: + cmd = ' '.join([config.CC, config.COMMON_FLAGS, config.COMPILE_FLAGS, source_path, '-c', '-o', object_path]) + if not os.path.exists(os.path.dirname(object_path)): + os.makedirs(os.path.dirname(object_path)) + source_hash = get_hash(source_path) + if (header_changed or not os.path.exists(object_path) + or source_path not in hash_dict or hash_dict[source_path] != source_hash): + todo_list.append((source_path, source_hash, cmd)) + continue + + # Running compilation processes + if not os.path.exists(config.OBJECT_DIR): + os.makedirs(config.OBJECT_DIR) + error_path = None + if arguments.j > 1: # Multi-process + jobs: List[subprocess.Popen] = [] + for source_path, source_hash, cmd in todo_list: + print(cmd) + jobs.insert(0, (source_path, source_hash, subprocess.Popen(cmd, shell=True))) # FIFO style (will be poped) + if len(jobs) >= arguments.j: # If jobs count is maxed we wait for the oldest one to finished + job_path, job_hash, oldest_job = jobs.pop() + oldest_job.wait() + if oldest_job.returncode != 0: + error_path = job_path + break + hash_dict[job_path] = job_hash # Update hash if no error + if error_path is None: + for job_path, job_hash, job in jobs: # Wait the last jobs to finish + job.wait() + if job.returncode != 0: + error_path = job_path + break + hash_dict[job_path] = job_hash # Update hash if no error + else: # Single-process + for source_path, source_hash, cmd in todo_list: + print(cmd) + complete = subprocess.run(cmd, check=False, shell=True) + if complete.returncode != 0: + error_path = source_path + break + hash_dict[source_path] = source_hash # Update hash if no error + if error_path: + print(f'Error compiling {error_path}') + with open(os.path.join(config.OBJECT_DIR, 'hash.json'), 'w') as hash_file: + hash_file.write(json.dumps(hash_dict, indent=1)) + sys.exit(1) + + # Running linking processes + if not os.path.exists(config.BIN_DIR): + os.makedirs(config.BIN_DIR) + for app_name in config.APPS: + app_path = os.path.join(config.BIN_DIR, app_name) + app_objects = [file_name + '.o' for file_name in config.APPS if file_name != app_name] + object_files = [object_path for _, object_path in compile_list + if os.path.basename(object_path) not in app_objects] + cmd = ' '.join([config.CC, config.COMMON_FLAGS, *object_files, '-o', app_path, config.LINK_FLAGS]) + print(cmd) + complete = subprocess.run(cmd, check=False, shell=True) + if complete.returncode != 0: + error_path = app_name + if error_path: + with open(os.path.join(config.OBJECT_DIR, 'hash.json'), 'w') as hash_file: + hash_file.write(json.dumps(hash_dict, indent=1)) + print('Error linking f{error_path}') + sys.exit(1) + + # Updating header hashes only if everything compiled and linked correctly + for header_path in header_hash_dict: + hash_dict[header_path] = header_hash_dict[header_path] + with open(os.path.join(config.OBJECT_DIR, 'hash.json'), 'w') as hash_file: + hash_file.write(json.dumps(hash_dict, indent=1))