shuriken: Add support for "import std"

shuriken.yaml

@@ -5,7 +5,6 @@ defaults:
     -Wextra
     -pedantic
     -Ivendor/RmlUi/Include
-
   linker_args: >-
     -lSDL3
     -lSDL3_image
@@ -13,6 +12,8 @@ defaults:
     vendor/RmlUi/Build/librmlui_debugger.a
     vendor/RmlUi/Build/librmlui.a
 
+enable_import_std: true
+
 default_target: linux
 
 targets:

tools/shuriken.py

@@ -64,6 +64,7 @@ class TargetConfig:
 class ShurikenConfig:
     __path_fields = ['source_dir', 'build_dir']
     __list_fields = []
+    __bool_fields = ['enable_import_std']
 
     source_dir: Path = Path('src')
     build_dir: Path = Path('build')
@@ -76,6 +77,8 @@ class ShurikenConfig:
         linker_args='',
     ))
 
+    enable_import_std: bool = False
+
     default_target: str = ''
     targets: dict[str, TargetConfig] = field(default_factory=dict)
 
@@ -86,13 +89,13 @@ class ShurikenConfig:
                 continue
 
             if key == 'defaults':
-                assert (type(value) is dict)
+                assert type(value) is dict
                 self.defaults.update(value)
             elif key == 'targets':
-                assert (type(value) is dict)
+                assert type(value) is dict
                 for target_name, target_config in value.items():
-                    assert (type(target_name) is str and len(target_name) > 0)
+                    assert type(target_name) is str and len(target_name) > 0
-                    assert (type(target_config) is dict or target_config is None)
+                    assert type(target_config) is dict or target_config is None
 
                     # Ignore "hidden" targets starting with a dot (can be used for YAML anchors)
                     if target_name[0] == '.':
@@ -105,8 +108,11 @@ class ShurikenConfig:
             elif key in self.__path_fields:
                 setattr(self, key, Path(value))
             elif key in self.__list_fields:
-                assert (type(value) is list)
+                assert type(value) is list
                 setattr(self, key, list(str(item) for item in value))
+            elif key in self.__bool_fields:
+                assert type(value) is bool
+                setattr(self, key, value)
             else:
                 setattr(self, key, str(value))
 
@@ -484,13 +490,14 @@ class ShurikenCli:
         print('ninja_dyndep_version = 1\n')
         for out, deps in build_dependencies.items():
             if deps:
-                print(f'build {out}: dyndep | {" ".join(deps)}')
+                print(f'build {out}: dyndep | {" ".join(str(dep) for dep in deps)}')
             else:
                 print(f'build {out}: dyndep')
 
 
 CompileTargetCpp = namedtuple('CompileTargetCpp', ['src', 'obj'])
 CompileTargetCppm = namedtuple('CompileTargetCppm', ['src', 'obj', 'pcm'])
+CompileTargetStdModule = namedtuple('CompileTargetStdModule', ['src', 'pcm'])
 
 
 class NinjaFileGenerator:
@@ -500,6 +507,7 @@ class NinjaFileGenerator:
 
     compile_targets_cpp: list[CompileTargetCpp]
     compile_targets_cppm: list[CompileTargetCppm]
+    compile_targets_std_module: list[CompileTargetStdModule]
     link_target_obj_files: list[str]
 
     def __init__(self, *, config: ShurikenConfig, target: str):
@@ -517,6 +525,7 @@ class NinjaFileGenerator:
     def collect_compile_targets(self) -> None:
         self.compile_targets_cpp = []
         self.compile_targets_cppm = []
+        self.compile_targets_std_module = []
 
         # Compile targets for .cpp files
         for src_file in self.find_source_files('**/*.cpp'):
@@ -529,12 +538,24 @@ class NinjaFileGenerator:
         # Compile targets for .cppm files
         for src_file in self.find_source_files('**/*.cppm'):
             relative_name = src_file.relative_to(self.config.source_dir)
+            module_name = '.'.join(relative_name.with_suffix('.pcm').parts)
             self.compile_targets_cppm.append(CompileTargetCppm(
                 src=str(src_file),
                 obj=f"$builddir/obj/{relative_name.with_suffix('.o')}",
-                pcm=f"$builddir/pcm/{'.'.join(relative_name.with_suffix('.pcm').parts)}",
+                pcm=f"$builddir/pcm/{module_name}",
             ))
 
+        # Compile targets for standard library modules ("import std")
+        if self.config.enable_import_std:
+            # NOTE: Add std.compat to the list (here and a few other places) if we ever need it. For now, std is enough.
+            std_module_source_finder = StdModuleSourceFinder(config=self.config, target=self.target)
+            std_module_sources = std_module_source_finder.find_module_sources(['std'])
+            for module_name, module_src_path in std_module_sources.items():
+                self.compile_targets_std_module.append(CompileTargetStdModule(
+                    src=str(module_src_path),
+                    pcm=f"$builddir/pcm/{module_name}.pcm",
+                ))
+
         # Link target for output file (executable): gather all object files
         self.link_target_obj_files = sorted(
             target.obj for target in self.compile_targets_cpp + self.compile_targets_cppm
@@ -560,7 +581,9 @@ compdb_file = {build_dir}/compile_commands.json
 dyndep_file = $builddir/.ninja_dyndep
 
 # Compiler flags
-cpp_flags = -std={target_config.cpp_standard} -fprebuilt-module-path=$builddir/pcm/ {target_config.merged_cpp_flags}
+cpp_base_flags = -std={target_config.cpp_standard} -fprebuilt-module-path=$builddir/pcm/
+cpp_flags = $cpp_base_flags {target_config.merged_cpp_flags}
+cpp_std_module_flags = $cpp_base_flags -Wno-reserved-module-identifier -x c++-module
 linker_flags = {target_config.merged_linker_flags}
 linker_args = {target_config.merged_linker_args}
 
@@ -574,6 +597,10 @@ rule cpp
 rule cppm
     command = {target_config.cpp_compiler} $cpp_flags -c $in -o $out -fmodule-output=$pcm_out
 
+# Rule to compile a standard library module to a .pcm file (e.g. "import std")
+rule std_module
+    command = {target_config.cpp_compiler} $cpp_std_module_flags --precompile $in -o $out
+
 # Rule to link several .o files to an executable
 rule link
     command = {target_config.cpp_compiler} $linker_flags -o $out $in $linker_args
@@ -611,9 +638,10 @@ default $builddir/{target_config.output_file}
 """.strip()
 
     def render_compile_targets(self) -> str:
-        rendered_targets_cpp = [self.render_compile_target_cpp(target) for target in self.compile_targets_cpp]
+        rendered_targets = list(map(self.render_compile_target_std_module, self.compile_targets_std_module))
-        rendered_targets_cppm = [self.render_compile_target_cppm(target) for target in self.compile_targets_cppm]
+        rendered_targets += list(map(self.render_compile_target_cpp, self.compile_targets_cpp))
-        return '\n\n'.join(rendered_targets_cpp + rendered_targets_cppm)
+        rendered_targets += list(map(self.render_compile_target_cppm, self.compile_targets_cppm))
+        return '\n\n'.join(rendered_targets)
 
     @staticmethod
     def render_compile_target_cpp(target: CompileTargetCpp) -> str:
@@ -630,23 +658,36 @@ build {target.obj} | {target.pcm}: cppm {target.src} || $dyndep_file
     pcm_out = {target.pcm}
 """.strip()
 
+    @staticmethod
+    def render_compile_target_std_module(target: CompileTargetStdModule) -> str:
+        return f"""
+build {target.pcm}: std_module {target.src} || $dyndep_file
+    dyndep = $dyndep_file
+""".strip()
+
 
 class BuildDependencyParser:
     config: ShurikenConfig
     target: str
 
-    def __init__(self, config: ShurikenConfig, target: str):
+    def __init__(self, *, config: ShurikenConfig, target: str):
         self.config = config
         self.target = target
 
-    def parse_build_dependencies(self, parsed_p1689: dict[str, Any]) -> dict[str, list[str]]:
+    def parse_build_dependencies(self, parsed_p1689: dict[str, Any]) -> dict[Path, list[Path]]:
         # Shortcut variables
         source_dir = self.config.source_dir
         target_build_dir = self.config.build_dir / self.target
         default_target_build_dir = self.config.build_dir / self.config.default_target
 
+        # Map of module names to PCM file paths
+        module_map: dict[str, Path] = {}
+
+        # Add standard library modules to module map (if "import std" support is enabled)
+        if self.config.enable_import_std:
+            module_map['std'] = Path(target_build_dir, 'pcm', 'std.pcm')
+
         # Validate module name and path convention, construct map of module names to PCM file paths
-        module_map = {}
         for rule in parsed_p1689['rules']:
             provides = rule.get('provides', [])
             if provides:
@@ -654,15 +695,23 @@ class BuildDependencyParser:
                 assert provides[0]['is-interface'] is True, f"Rule provides non-interface module: {rule}"
 
                 module_name: str = provides[0]['logical-name']
-                assert module_name not in module_map, f"Module {module_name} is provided more than once: {rule}"
+                assert module_name not in module_map, f'Module "{module_name}" is provided more than once: {rule}'
 
                 expected_source_path = str(Path(source_dir, module_name.replace('.', '/') + '.cppm'))
-                assert provides[0]['source-path'] == expected_source_path, f"Module name does not match source path: {rule}"
+                assert provides[0]['source-path'] == expected_source_path, \
+                    f"Module name does not match source path: {rule}"
 
-                module_map[module_name] = str(Path(target_build_dir, 'pcm', module_name + '.pcm'))
+                module_map[module_name] = Path(target_build_dir, 'pcm', module_name + '.pcm')
+
+        # Collect all module dependencies to generate dyndeps
+        build_dependencies: dict[Path, list[Path]] = {}
+
+        # If "import std" support is enabled, we need to add std to the build dependencies, even though it doesn't have
+        # any dependencies itself. This is needed, otherwise the module won't be built automatically.
+        if self.config.enable_import_std:
+            build_dependencies[target_build_dir / 'pcm/std.pcm'] = []
 
         # Parse rules and get module dependencies
-        build_dependencies = {}
         for rule in parsed_p1689['rules']:
             # Compilation database contains "build/{default_target}" paths, we need to map them to "build/{target}"
             out = target_build_dir / Path(rule['primary-output']).relative_to(default_target_build_dir)
@@ -670,11 +719,65 @@ class BuildDependencyParser:
             build_dependencies[out] = []
             for require in rule.get('requires', []):
                 module_name = require['logical-name']
-                assert module_name in module_map, f'Module {module_name} not found in module map'
+                assert module_name in module_map, f'Module "{module_name}" not found in module map'
                 build_dependencies[out].append(module_map[module_name])
 
         return build_dependencies
 
 
+class StdModuleSourceFinder:
+    cpp_compiler: str
+
+    def __init__(self, *, config: ShurikenConfig, target: str):
+        self.cpp_compiler = config.get_merged_target_config(target).cpp_compiler
+
+    def find_module_sources(self, module_names: list[str]) -> dict[str, Path]:
+        # Get normalized path of standard library "module manifest" (JSON file containing source paths for std modules)
+        module_manifest_path = self.get_module_manifest_path()
+        module_manifest_base_dir = module_manifest_path.parent
+
+        # Parse module manifest JSON file
+        with module_manifest_path.open('r') as f:
+            module_manifest_json = json.load(f)
+
+        # Collect the module source paths
+        std_module_sources: dict[str, Path] = {}
+        for modules_item in module_manifest_json['modules']:
+            # Ignore modules we don't need (probably just std.compat)
+            module_name = modules_item['logical-name']
+            if module_name not in module_names:
+                continue
+
+            # The source path is usually relative to the module manifest file, convert to absolute path
+            module_src_path = Path(modules_item['source-path'])
+            if not module_src_path.is_absolute():
+                module_src_path = module_manifest_base_dir / module_src_path
+            if not module_src_path.is_file():
+                raise Exception(f'Source path for module "{module_name}" is not a file: {module_src_path}')
+
+            # Resolve symlinks and normalize path
+            std_module_sources[module_name] = module_src_path.resolve()
+
+        return std_module_sources
+
+    def get_module_manifest_path(self) -> Path:
+        # Use compiler to get the path to the module manifest JSON file
+        clang_cmd = [self.cpp_compiler, '-print-library-module-manifest-path']
+        clang_cmd_result = subprocess.run(clang_cmd, capture_output=True)
+
+        if clang_cmd_result.returncode != 0:
+            cmd_stderr = clang_cmd_result.stderr.decode().strip()
+            raise Exception(f'Subprocess "{" ".join(clang_cmd)}" failed. stderr:\n{cmd_stderr}')
+
+        # Read path from subprocess stdout
+        module_manifest_path = Path(clang_cmd_result.stdout.decode().strip())
+
+        if not module_manifest_path.is_file():
+            raise Exception(f'Could not locate stdlib module manifest: {module_manifest_path}')
+
+        # Resolve symlinks and normalize path
+        return module_manifest_path.resolve()
+
+
 if __name__ == '__main__':
     ShurikenCli().run()