umake/umake.py

#! python3

from argparse import ArgumentParser
import hashlib
import json
import os
from pathlib import Path
import subprocess
import shutil
import sys
from typing import List


class Config:
    CC = 'g++'                             # Compiler to call
    APPS = ['app_name']                    # Output binaries (need to be found as .cpp directly in SOURCE_DIR)
    JOB_COUNT = int(os.cpu_count() * 0.8)  # Concurent jobs (multi-processing)

    BIN_DIR = Path('bin')                # Output directory (binaries)
    INCLUDE_DIR = Path('include')  # Include directory (header files)
    OBJECT_DIR = Path('obj')       # Temporary directory (object files)
    SOURCE_DIR = Path('src')       # Source directories

    COMMON_FLAGS = '-std=c++17'               # Flags used for comiling and linking
    COMMON_DEBUG_FLAGS = '-g'                 # Flags added in debug mode
    COMMON_RELEASE_FLAGS = '-O2 -flto'        # Flags added in release mode
    COMPILE_FLAGS = f'-Wall -I{INCLUDE_DIR}'  # Flags added for compiling (recommandation : `pkg-config --cflags`)
    LINK_FLAGS = ''                           # Flags added for linking (recommandation : `pkg-config --libs`)

    CPP_HEADERS = INCLUDE_DIR.rglob('*.hpp')
    CPP_SOURCES = SOURCE_DIR.rglob('*.cpp')


def get_hash(path: Path) -> str:
    with open(path, 'r') as hashing_file:
        hash_obj = hashlib.md5()
        hash_obj.update(hashing_file.read().encode())
    return hash_obj.hexdigest()


def make(config: Config):
    parser = ArgumentParser()
    parser.add_argument('-j', type=int, default=config.JOB_COUNT, help='Jobs count (multi-processing)')
    parser.add_argument('--type', default='production', help='Compilation type (release, debug). Default=release')
    parser.add_argument('--clean', action='store_true', help='Clean all file instead of building')
    arguments = parser.parse_args()

    # Clean action
    if arguments.clean:
        shutil.rmtree(config.OBJECT_DIR, ignore_errors=True)
        shutil.rmtree(config.BIN_DIR, ignore_errors=True)
        return

    # Update flags and directories for mode debug/release
    if arguments.type == 'debug':
        config.COMMON_FLAGS += ' ' + config.COMMON_DEBUG_FLAGS
        config.OBJECT_DIR = config.OBJECT_DIR / 'debug'
    else:
        config.COMMON_FLAGS += ' ' + config.COMMON_RELEASE_FLAGS
        config.OBJECT_DIR = config.OBJECT_DIR / 'release'

    # Create list of source to process (tuple[source_path, object_path])
    compile_list = [(Path(source_file),
                     config.OBJECT_DIR / source_file.parent.relative_to(config.SOURCE_DIR) / (source_file.stem + '.o'))
                    for source_file in config.CPP_SOURCES]
    todo_list = []

    hash_dict = {}
    if (config.OBJECT_DIR / 'hash.json').exists():
        with open(config.OBJECT_DIR / 'hash.json', 'r') as hash_file:
            hash_dict = json.loads(hash_file.read())

    # Check header files (if any modification is done all source file will be processed to avoid any problem)
    header_changed = False
    header_hash_dict = {}
    for header_path in config.CPP_HEADERS:
        header_hash = get_hash(header_path)
        if str(header_path) not in hash_dict or hash_dict[str(header_path)] != header_hash:
            header_hash_dict[str(header_path)] = header_hash
            header_changed = True

    # Check source file and generate compilation commands to execute
    for source_path, object_path in compile_list:
        cmd = ' '.join([config.CC, config.COMMON_FLAGS, config.COMPILE_FLAGS,
                        str(source_path), '-c', '-o', str(object_path)])
        if not object_path.parent.exists():
            object_path.parent.mkdir(parents=True)
        source_hash = get_hash(source_path)
        if (header_changed or not object_path.exists()
                or str(source_path) not in hash_dict or hash_dict[str(source_path)] != source_hash):
            todo_list.append((source_path, source_hash, cmd))
            continue

    # Running compilation processes
    if not os.path.exists(config.OBJECT_DIR):
        os.makedirs(config.OBJECT_DIR)
    error_path = None
    if arguments.j > 1:  # Multi-process
        jobs: List[subprocess.Popen] = []
        for source_path, source_hash, cmd in todo_list:
            print(cmd)
            jobs.insert(0, (source_path, source_hash, subprocess.Popen(cmd, shell=True)))  # FIFO style (will be poped)
            if len(jobs) >= arguments.j:  # If jobs count is maxed we wait for the oldest one to finished
                job_path, job_hash, oldest_job = jobs.pop()
                oldest_job.wait()
                if oldest_job.returncode != 0:
                    error_path = job_path
                    break
                hash_dict[str(job_path)] = job_hash  # Update hash if no error
        if error_path is None:
            for job_path, job_hash, job in jobs:  # Wait the last jobs to finish
                job.wait()
                if job.returncode != 0:
                    error_path = job_path
                    break
                hash_dict[str(job_path)] = job_hash  # Update hash if no error
    else:  # Single-process
        for source_path, source_hash, cmd in todo_list:
            print(cmd)
            complete = subprocess.run(cmd, check=False, shell=True)
            if complete.returncode != 0:
                error_path = source_path
                break
            hash_dict[str(source_path)] = source_hash  # Update hash if no error
    if error_path:
        print(f'Error compiling {error_path}')
        with open(config.OBJECT_DIR / 'hash.json', 'w') as hash_file:
            hash_file.write(json.dumps(hash_dict, indent=1))
        sys.exit(1)

    # Running linking processes
    if not config.BIN_DIR.exists():
        config.BIN_DIR.mkdir(parents=True)
    all_app_objects = [config.OBJECT_DIR / Path(app_path).parent / (Path(app_path).stem + '.o')
                       for app_path in config.APPS]
    for app_path, app_object_path in zip(config.APPS, all_app_objects):
        if 'IGNORE_APPS' in config.__dict__ and app_path in config.IGNORE_APPS:
            continue
        bin_path = config.BIN_DIR / app_path
        if not bin_path.parent.exists():
            bin_path.parent.mkdir(parents=True)
        object_files = [str(object_path) for _, object_path in compile_list
                        if object_path not in all_app_objects]
        object_files.append(str(app_object_path))
        cmd = ' '.join([config.CC, config.COMMON_FLAGS, *object_files, '-o', str(bin_path), config.LINK_FLAGS])
        complete = subprocess.run(cmd, check=False, shell=True)
        if complete.returncode != 0:
            error_path = app_path
    if error_path:
        with open(os.path.join(config.OBJECT_DIR, 'hash.json'), 'w') as hash_file:
            hash_file.write(json.dumps(hash_dict, indent=1))
        print(f'Error linking {error_path}')
        sys.exit(1)

    # Updating header hashes only if everything compiled and linked correctly
    for header_path in header_hash_dict:
        hash_dict[header_path] = header_hash_dict[header_path]
    with open(config.OBJECT_DIR / 'hash.json', 'w') as hash_file:
        hash_file.write(json.dumps(hash_dict, indent=1))