diff --git a/shuriken.yaml b/shuriken.yaml index fee3c69..0a07b3f 100644 --- a/shuriken.yaml +++ b/shuriken.yaml @@ -5,7 +5,6 @@ defaults: -Wextra -pedantic -Ivendor/RmlUi/Include - linker_args: >- -lSDL3 -lSDL3_image @@ -13,6 +12,8 @@ defaults: vendor/RmlUi/Build/librmlui_debugger.a vendor/RmlUi/Build/librmlui.a +enable_import_std: true + default_target: linux targets: diff --git a/tools/shuriken.py b/tools/shuriken.py index fcb6402..955c672 100755 --- a/tools/shuriken.py +++ b/tools/shuriken.py @@ -64,6 +64,7 @@ class TargetConfig: class ShurikenConfig: __path_fields = ['source_dir', 'build_dir'] __list_fields = [] + __bool_fields = ['enable_import_std'] source_dir: Path = Path('src') build_dir: Path = Path('build') @@ -76,6 +77,8 @@ class ShurikenConfig: linker_args='', )) + enable_import_std: bool = False + default_target: str = '' targets: dict[str, TargetConfig] = field(default_factory=dict) @@ -86,13 +89,13 @@ class ShurikenConfig: continue if key == 'defaults': - assert (type(value) is dict) + assert type(value) is dict self.defaults.update(value) elif key == 'targets': - assert (type(value) is dict) + assert type(value) is dict for target_name, target_config in value.items(): - assert (type(target_name) is str and len(target_name) > 0) - assert (type(target_config) is dict or target_config is None) + assert type(target_name) is str and len(target_name) > 0 + assert type(target_config) is dict or target_config is None # Ignore "hidden" targets starting with a dot (can be used for YAML anchors) if target_name[0] == '.': @@ -105,8 +108,11 @@ class ShurikenConfig: elif key in self.__path_fields: setattr(self, key, Path(value)) elif key in self.__list_fields: - assert (type(value) is list) + assert type(value) is list setattr(self, key, list(str(item) for item in value)) + elif key in self.__bool_fields: + assert type(value) is bool + setattr(self, key, value) else: setattr(self, key, str(value)) @@ -484,13 +490,14 @@ class ShurikenCli: print('ninja_dyndep_version = 1\n') for out, deps in build_dependencies.items(): if deps: - print(f'build {out}: dyndep | {" ".join(deps)}') + print(f'build {out}: dyndep | {" ".join(str(dep) for dep in deps)}') else: print(f'build {out}: dyndep') CompileTargetCpp = namedtuple('CompileTargetCpp', ['src', 'obj']) CompileTargetCppm = namedtuple('CompileTargetCppm', ['src', 'obj', 'pcm']) +CompileTargetStdModule = namedtuple('CompileTargetStdModule', ['src', 'pcm']) class NinjaFileGenerator: @@ -500,6 +507,7 @@ class NinjaFileGenerator: compile_targets_cpp: list[CompileTargetCpp] compile_targets_cppm: list[CompileTargetCppm] + compile_targets_std_module: list[CompileTargetStdModule] link_target_obj_files: list[str] def __init__(self, *, config: ShurikenConfig, target: str): @@ -517,6 +525,7 @@ class NinjaFileGenerator: def collect_compile_targets(self) -> None: self.compile_targets_cpp = [] self.compile_targets_cppm = [] + self.compile_targets_std_module = [] # Compile targets for .cpp files for src_file in self.find_source_files('**/*.cpp'): @@ -529,12 +538,24 @@ class NinjaFileGenerator: # Compile targets for .cppm files for src_file in self.find_source_files('**/*.cppm'): relative_name = src_file.relative_to(self.config.source_dir) + module_name = '.'.join(relative_name.with_suffix('.pcm').parts) self.compile_targets_cppm.append(CompileTargetCppm( src=str(src_file), obj=f"$builddir/obj/{relative_name.with_suffix('.o')}", - pcm=f"$builddir/pcm/{'.'.join(relative_name.with_suffix('.pcm').parts)}", + pcm=f"$builddir/pcm/{module_name}", )) + # Compile targets for standard library modules ("import std") + if self.config.enable_import_std: + # NOTE: Add std.compat to the list (here and a few other places) if we ever need it. For now, std is enough. + std_module_source_finder = StdModuleSourceFinder(config=self.config, target=self.target) + std_module_sources = std_module_source_finder.find_module_sources(['std']) + for module_name, module_src_path in std_module_sources.items(): + self.compile_targets_std_module.append(CompileTargetStdModule( + src=str(module_src_path), + pcm=f"$builddir/pcm/{module_name}.pcm", + )) + # Link target for output file (executable): gather all object files self.link_target_obj_files = sorted( target.obj for target in self.compile_targets_cpp + self.compile_targets_cppm @@ -560,7 +581,9 @@ compdb_file = {build_dir}/compile_commands.json dyndep_file = $builddir/.ninja_dyndep # Compiler flags -cpp_flags = -std={target_config.cpp_standard} -fprebuilt-module-path=$builddir/pcm/ {target_config.merged_cpp_flags} +cpp_base_flags = -std={target_config.cpp_standard} -fprebuilt-module-path=$builddir/pcm/ +cpp_flags = $cpp_base_flags {target_config.merged_cpp_flags} +cpp_std_module_flags = $cpp_base_flags -Wno-reserved-module-identifier -x c++-module linker_flags = {target_config.merged_linker_flags} linker_args = {target_config.merged_linker_args} @@ -574,6 +597,10 @@ rule cpp rule cppm command = {target_config.cpp_compiler} $cpp_flags -c $in -o $out -fmodule-output=$pcm_out +# Rule to compile a standard library module to a .pcm file (e.g. "import std") +rule std_module + command = {target_config.cpp_compiler} $cpp_std_module_flags --precompile $in -o $out + # Rule to link several .o files to an executable rule link command = {target_config.cpp_compiler} $linker_flags -o $out $in $linker_args @@ -611,9 +638,10 @@ default $builddir/{target_config.output_file} """.strip() def render_compile_targets(self) -> str: - rendered_targets_cpp = [self.render_compile_target_cpp(target) for target in self.compile_targets_cpp] - rendered_targets_cppm = [self.render_compile_target_cppm(target) for target in self.compile_targets_cppm] - return '\n\n'.join(rendered_targets_cpp + rendered_targets_cppm) + rendered_targets = list(map(self.render_compile_target_std_module, self.compile_targets_std_module)) + rendered_targets += list(map(self.render_compile_target_cpp, self.compile_targets_cpp)) + rendered_targets += list(map(self.render_compile_target_cppm, self.compile_targets_cppm)) + return '\n\n'.join(rendered_targets) @staticmethod def render_compile_target_cpp(target: CompileTargetCpp) -> str: @@ -630,23 +658,36 @@ build {target.obj} | {target.pcm}: cppm {target.src} || $dyndep_file pcm_out = {target.pcm} """.strip() + @staticmethod + def render_compile_target_std_module(target: CompileTargetStdModule) -> str: + return f""" +build {target.pcm}: std_module {target.src} || $dyndep_file + dyndep = $dyndep_file +""".strip() + class BuildDependencyParser: config: ShurikenConfig target: str - def __init__(self, config: ShurikenConfig, target: str): + def __init__(self, *, config: ShurikenConfig, target: str): self.config = config self.target = target - def parse_build_dependencies(self, parsed_p1689: dict[str, Any]) -> dict[str, list[str]]: + def parse_build_dependencies(self, parsed_p1689: dict[str, Any]) -> dict[Path, list[Path]]: # Shortcut variables source_dir = self.config.source_dir target_build_dir = self.config.build_dir / self.target default_target_build_dir = self.config.build_dir / self.config.default_target + # Map of module names to PCM file paths + module_map: dict[str, Path] = {} + + # Add standard library modules to module map (if "import std" support is enabled) + if self.config.enable_import_std: + module_map['std'] = Path(target_build_dir, 'pcm', 'std.pcm') + # Validate module name and path convention, construct map of module names to PCM file paths - module_map = {} for rule in parsed_p1689['rules']: provides = rule.get('provides', []) if provides: @@ -654,15 +695,23 @@ class BuildDependencyParser: assert provides[0]['is-interface'] is True, f"Rule provides non-interface module: {rule}" module_name: str = provides[0]['logical-name'] - assert module_name not in module_map, f"Module {module_name} is provided more than once: {rule}" + assert module_name not in module_map, f'Module "{module_name}" is provided more than once: {rule}' expected_source_path = str(Path(source_dir, module_name.replace('.', '/') + '.cppm')) - assert provides[0]['source-path'] == expected_source_path, f"Module name does not match source path: {rule}" + assert provides[0]['source-path'] == expected_source_path, \ + f"Module name does not match source path: {rule}" - module_map[module_name] = str(Path(target_build_dir, 'pcm', module_name + '.pcm')) + module_map[module_name] = Path(target_build_dir, 'pcm', module_name + '.pcm') + + # Collect all module dependencies to generate dyndeps + build_dependencies: dict[Path, list[Path]] = {} + + # If "import std" support is enabled, we need to add std to the build dependencies, even though it doesn't have + # any dependencies itself. This is needed, otherwise the module won't be built automatically. + if self.config.enable_import_std: + build_dependencies[target_build_dir / 'pcm/std.pcm'] = [] # Parse rules and get module dependencies - build_dependencies = {} for rule in parsed_p1689['rules']: # Compilation database contains "build/{default_target}" paths, we need to map them to "build/{target}" out = target_build_dir / Path(rule['primary-output']).relative_to(default_target_build_dir) @@ -670,11 +719,65 @@ class BuildDependencyParser: build_dependencies[out] = [] for require in rule.get('requires', []): module_name = require['logical-name'] - assert module_name in module_map, f'Module {module_name} not found in module map' + assert module_name in module_map, f'Module "{module_name}" not found in module map' build_dependencies[out].append(module_map[module_name]) return build_dependencies +class StdModuleSourceFinder: + cpp_compiler: str + + def __init__(self, *, config: ShurikenConfig, target: str): + self.cpp_compiler = config.get_merged_target_config(target).cpp_compiler + + def find_module_sources(self, module_names: list[str]) -> dict[str, Path]: + # Get normalized path of standard library "module manifest" (JSON file containing source paths for std modules) + module_manifest_path = self.get_module_manifest_path() + module_manifest_base_dir = module_manifest_path.parent + + # Parse module manifest JSON file + with module_manifest_path.open('r') as f: + module_manifest_json = json.load(f) + + # Collect the module source paths + std_module_sources: dict[str, Path] = {} + for modules_item in module_manifest_json['modules']: + # Ignore modules we don't need (probably just std.compat) + module_name = modules_item['logical-name'] + if module_name not in module_names: + continue + + # The source path is usually relative to the module manifest file, convert to absolute path + module_src_path = Path(modules_item['source-path']) + if not module_src_path.is_absolute(): + module_src_path = module_manifest_base_dir / module_src_path + if not module_src_path.is_file(): + raise Exception(f'Source path for module "{module_name}" is not a file: {module_src_path}') + + # Resolve symlinks and normalize path + std_module_sources[module_name] = module_src_path.resolve() + + return std_module_sources + + def get_module_manifest_path(self) -> Path: + # Use compiler to get the path to the module manifest JSON file + clang_cmd = [self.cpp_compiler, '-print-library-module-manifest-path'] + clang_cmd_result = subprocess.run(clang_cmd, capture_output=True) + + if clang_cmd_result.returncode != 0: + cmd_stderr = clang_cmd_result.stderr.decode().strip() + raise Exception(f'Subprocess "{" ".join(clang_cmd)}" failed. stderr:\n{cmd_stderr}') + + # Read path from subprocess stdout + module_manifest_path = Path(clang_cmd_result.stdout.decode().strip()) + + if not module_manifest_path.is_file(): + raise Exception(f'Could not locate stdlib module manifest: {module_manifest_path}') + + # Resolve symlinks and normalize path + return module_manifest_path.resolve() + + if __name__ == '__main__': ShurikenCli().run()