diff --git a/docs/ChangeLog/20250525/pr25237.md b/docs/ChangeLog/20250525/pr25237.md new file mode 100644 index 00000000000..ca09e93ebc8 --- /dev/null +++ b/docs/ChangeLog/20250525/pr25237.md @@ -0,0 +1,3 @@ +# Deprecation of `qmk generate-compilation-database` + +This command has been deprecated as it cannot take into account configurables such as [converters](/feature_converters) or environment variables normally specified on the command line; please use the `--compiledb` flag with `qmk compile` instead. diff --git a/docs/cli_commands.md b/docs/cli_commands.md index d17b0eda237..c6a7ffe1918 100644 --- a/docs/cli_commands.md +++ b/docs/cli_commands.md @@ -17,7 +17,7 @@ qmk compile [-c] **Usage for Keymaps**: ``` -qmk compile [-c] [-e =] [-j ] -kb -km +qmk compile [-c] [-e =] [-j ] [--compiledb] -kb -km ``` **Usage in Keyboard Directory**: @@ -84,6 +84,25 @@ The `num_jobs` argument determines the maximum number of jobs that can be used. qmk compile -j 0 -kb ``` +**Compilation Database**: + +Creates a `compile_commands.json` file. + +Does your IDE/editor use a language server but doesn't _quite_ find all the necessary include files? Do you hate red squigglies? Do you wish your editor could figure out `#include QMK_KEYBOARD_H`? You might need a [compilation database](https://clang.llvm.org/docs/JSONCompilationDatabase.html)! Compiling using this argument can create this for you. + +**Example:** + +``` +$ cd ~/qmk_firmware/keyboards/gh60/satan/keymaps/colemak +$ qmk compile --compiledb +Ψ Making clean +Ψ Gathering build instructions from make ........ +Ψ Found 63 compile commands +Ψ Writing build database to /Users/you/src/qmk_firmware/compile_commands.json +Ψ Compiling keymap with make ........ +... build log continues ... +``` + ## `qmk flash` This command is similar to `qmk compile`, but can also target a bootloader. The bootloader is optional, and is set to `:flash` by default. To specify a different bootloader, use `-bl `. Visit the [Flashing Firmware](flashing) guide for more details of the available bootloaders. @@ -694,33 +713,6 @@ qmk format-c qmk format-c -b branch_name ``` -## `qmk generate-compilation-database` - -**Usage**: - -``` -qmk generate-compilation-database [-kb KEYBOARD] [-km KEYMAP] -``` - -Creates a `compile_commands.json` file. - -Does your IDE/editor use a language server but doesn't _quite_ find all the necessary include files? Do you hate red squigglies? Do you wish your editor could figure out `#include QMK_KEYBOARD_H`? You might need a [compilation database](https://clang.llvm.org/docs/JSONCompilationDatabase.html)! The qmk tool can build this for you. - -This command needs to know which keyboard and keymap to build. It uses the same configuration options as the `qmk compile` command: arguments, current directory, and config files. - -**Example:** - -``` -$ cd ~/qmk_firmware/keyboards/gh60/satan/keymaps/colemak -$ qmk generate-compilation-database -Ψ Making clean -Ψ Gathering build instructions from make -n gh60/satan:colemak -Ψ Found 50 compile commands -Ψ Writing build database to /Users/you/src/qmk_firmware/compile_commands.json -``` - -Now open your dev environment and live a squiggly-free life. - ## `qmk docs` This command starts a local HTTP server which you can use for browsing or improving the docs, and provides live reload capability whilst editing. Default port is 8936. @@ -885,3 +877,13 @@ Run single test: ``` qmk test-c --test basic ``` + +## `qmk generate-compilation-database` + +**Usage**: + +``` +qmk generate-compilation-database [-kb KEYBOARD] [-km KEYMAP] +``` + +This command has been deprecated as it cannot take into account configurables such as [converters](/feature_converters) or environment variables normally specified on the command line; please use the `--compiledb` flag with `qmk compile` instead. diff --git a/docs/other_vscode.md b/docs/other_vscode.md index ab2b23e96b4..0b39746a6e9 100644 --- a/docs/other_vscode.md +++ b/docs/other_vscode.md @@ -112,7 +112,7 @@ Restart once you've installed any extensions. Using the [standard `compile_commands.json` database](https://clang.llvm.org/docs/JSONCompilationDatabase.html), we can get the VS code _clangd_ extension to use the correct includes and defines used for your keyboard and keymap. -1. Run `qmk generate-compilation-database -kb -km ` to generate the `compile_commands.json`. +1. Run `qmk compile -kb -km --compiledb` to generate the `compile_commands.json`. 1. Inside VS code, press Ctrl + Shift + P (macOS: Command + Shift + P) to open the command palette. 1. Start typing `clangd: Download Language Server` and select it when it appears. Note that this only needs to be done once on clangd extension installation, if it didn't already ask to do so. 1. Inside VS code, press Ctrl + Shift + P (macOS: Command + Shift + P) to open the command palette. diff --git a/lib/python/qmk/build_targets.py b/lib/python/qmk/build_targets.py index df5a5ffb42d..35a5f89f91a 100644 --- a/lib/python/qmk/build_targets.py +++ b/lib/python/qmk/build_targets.py @@ -12,6 +12,7 @@ from qmk.keyboard import keyboard_folder from qmk.info import keymap_json from qmk.keymap import locate_keymap from qmk.path import is_under_qmk_firmware, is_under_qmk_userspace, unix_style_path +from qmk.compilation_database import write_compilation_database # These must be kept in the order in which they're applied to $(TARGET) in the makefiles in order to ensure consistency. TARGET_FILENAME_MODIFIERS = ['FORCE_LAYOUT', 'CONVERT_TO'] @@ -150,7 +151,6 @@ class BuildTarget: def generate_compilation_database(self, build_target: str = None, skip_clean: bool = False, **env_vars) -> None: self.prepare_build(build_target=build_target, **env_vars) command = self.compile_command(build_target=build_target, dry_run=True, **env_vars) - from qmk.cli.generate.compilation_database import write_compilation_database # Lazy load due to circular references output_path = QMK_FIRMWARE / 'compile_commands.json' ret = write_compilation_database(command=command, output_path=output_path, skip_clean=skip_clean, **env_vars) if ret and output_path.exists() and HAS_QMK_USERSPACE: diff --git a/lib/python/qmk/cli/generate/compilation_database.py b/lib/python/qmk/cli/generate/compilation_database.py old mode 100755 new mode 100644 index b9c716bf0c5..339b53c2c2a --- a/lib/python/qmk/cli/generate/compilation_database.py +++ b/lib/python/qmk/cli/generate/compilation_database.py @@ -1,169 +1,9 @@ -"""Creates a compilation database for the given keyboard build. -""" - -import json -import os -import re -import shlex -import shutil -from functools import lru_cache -from pathlib import Path -from typing import Dict, Iterator, List, Union - -from milc import cli, MILC - -from qmk.commands import find_make -from qmk.constants import QMK_FIRMWARE -from qmk.decorators import automagic_keyboard, automagic_keymap -from qmk.keyboard import keyboard_completer, keyboard_folder -from qmk.keymap import keymap_completer -from qmk.build_targets import KeyboardKeymapBuildTarget +from milc import cli -@lru_cache(maxsize=10) -def system_libs(binary: str) -> List[Path]: - """Find the system include directory that the given build tool uses. - """ - cli.log.debug("searching for system library directory for binary: %s", binary) - - # Actually query xxxxxx-gcc to find its include paths. - if binary.endswith("gcc") or binary.endswith("g++"): - # (TODO): Remove 'stdin' once 'input' no longer causes issues under MSYS - result = cli.run([binary, '-E', '-Wp,-v', '-'], capture_output=True, check=True, stdin=None, input='\n') - paths = [] - for line in result.stderr.splitlines(): - if line.startswith(" "): - paths.append(Path(line.strip()).resolve()) - return paths - - return list(Path(binary).resolve().parent.parent.glob("*/include")) if binary else [] - - -@lru_cache(maxsize=10) -def cpu_defines(binary: str, compiler_args: str) -> List[str]: - cli.log.debug("gathering definitions for compilation: %s %s", binary, compiler_args) - if binary.endswith("gcc") or binary.endswith("g++"): - invocation = [binary, '-dM', '-E'] - if binary.endswith("gcc"): - invocation.extend(['-x', 'c']) - elif binary.endswith("g++"): - invocation.extend(['-x', 'c++']) - compiler_args = shlex.split(compiler_args) - invocation.extend(compiler_args) - invocation.append('-') - result = cli.run(invocation, capture_output=True, check=True, stdin=None, input='\n') - define_args = [] - for line in result.stdout.splitlines(): - line_args = line.split(' ', 2) - if len(line_args) == 3 and line_args[0] == '#define': - define_args.append(f'-D{line_args[1]}={line_args[2]}') - elif len(line_args) == 2 and line_args[0] == '#define': - define_args.append(f'-D{line_args[1]}') - return list(sorted(set(define_args))) - return [] - - -file_re = re.compile(r'printf "Compiling: ([^"]+)') -cmd_re = re.compile(r'LOG=\$\((.+?)&&') - - -def parse_make_n(f: Iterator[str]) -> List[Dict[str, str]]: - """parse the output of `make -n ` - - This function makes many assumptions about the format of your build log. - This happens to work right now for qmk. - """ - - state = 'start' - this_file = None - records = [] - for line in f: - if state == 'start': - m = file_re.search(line) - if m: - this_file = m.group(1) - state = 'cmd' - - if state == 'cmd': - assert this_file - m = cmd_re.search(line) - if m: - # we have a hit! - this_cmd = m.group(1) - args = shlex.split(this_cmd) - binary = shutil.which(args[0]) - compiler_args = set(filter(lambda x: x.startswith('-m') or x.startswith('-f'), args)) - for s in system_libs(binary): - args += ['-isystem', '%s' % s] - args.extend(cpu_defines(binary, ' '.join(shlex.quote(s) for s in compiler_args))) - new_cmd = ' '.join(shlex.quote(s) for s in args) - records.append({"directory": str(QMK_FIRMWARE.resolve()), "command": new_cmd, "file": this_file}) - state = 'start' - - return records - - -def write_compilation_database(keyboard: str = None, keymap: str = None, output_path: Path = QMK_FIRMWARE / 'compile_commands.json', skip_clean: bool = False, command: List[str] = None, **env_vars) -> bool: - # Generate the make command for a specific keyboard/keymap. - if not command: - from qmk.build_targets import KeyboardKeymapBuildTarget # Lazy load due to circular references - target = KeyboardKeymapBuildTarget(keyboard, keymap) - command = target.compile_command(dry_run=True, **env_vars) - - if not command: - cli.log.error('You must supply both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.') - cli.echo('usage: qmk generate-compilation-database [-kb KEYBOARD] [-km KEYMAP]') - return False - - # remove any environment variable overrides which could trip us up - env = os.environ.copy() - env.pop("MAKEFLAGS", None) - - # re-use same executable as the main make invocation (might be gmake) - if not skip_clean: - clean_command = [find_make(), "clean"] - cli.log.info('Making clean with {fg_cyan}%s', ' '.join(clean_command)) - cli.run(clean_command, capture_output=False, check=True, env=env) - - cli.log.info('Gathering build instructions from {fg_cyan}%s', ' '.join(command)) - - result = cli.run(command, capture_output=True, check=True, env=env) - db = parse_make_n(result.stdout.splitlines()) - if not db: - cli.log.error("Failed to parse output from make output:\n%s", result.stdout) - return False - - cli.log.info("Found %s compile commands", len(db)) - - cli.log.info(f"Writing build database to {output_path}") - output_path.write_text(json.dumps(db, indent=4)) - - return True - - -@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard\'s name') -@cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap\'s name') -@cli.subcommand('Create a compilation database.') -@automagic_keyboard -@automagic_keymap -def generate_compilation_database(cli: MILC) -> Union[bool, int]: - """Creates a compilation database for the given keyboard build. - - Does a make clean, then a make -n for this target and uses the dry-run output to create - a compilation database (compile_commands.json). This file can help some IDEs and - IDE-like editors work better. For more information about this: - - https://clang.llvm.org/docs/JSONCompilationDatabase.html - """ - # check both config domains: the magic decorator fills in `generate_compilation_database` but the user is - # more likely to have set `compile` in their config file. - current_keyboard = cli.config.generate_compilation_database.keyboard or cli.config.user.keyboard - current_keymap = cli.config.generate_compilation_database.keymap or cli.config.user.keymap - - if not current_keyboard: - cli.log.error('Could not determine keyboard!') - elif not current_keymap: - cli.log.error('Could not determine keymap!') - - target = KeyboardKeymapBuildTarget(current_keyboard, current_keymap) - return target.generate_compilation_database() +@cli.argument('-kb', '--keyboard', help='[unused] The keyboard\'s name') +@cli.argument('-km', '--keymap', help='[unused] The keymap\'s name') +@cli.subcommand('[deprecated] Create a compilation database.') +def generate_compilation_database(cli): + cli.log.error('This command is deprecated and has effectively been removed. Please use the `--compiledb` flag with `qmk compile` instead.') + return False diff --git a/lib/python/qmk/compilation_database.py b/lib/python/qmk/compilation_database.py new file mode 100755 index 00000000000..4c88dadbdd7 --- /dev/null +++ b/lib/python/qmk/compilation_database.py @@ -0,0 +1,137 @@ +"""Creates a compilation database for the given keyboard build. +""" + +import json +import os +import re +import shlex +import shutil +from functools import lru_cache +from pathlib import Path +from typing import Dict, Iterator, List + +from milc import cli + +from qmk.commands import find_make +from qmk.constants import QMK_FIRMWARE + + +@lru_cache(maxsize=10) +def system_libs(binary: str) -> List[Path]: + """Find the system include directory that the given build tool uses. + """ + cli.log.debug("searching for system library directory for binary: %s", binary) + + # Actually query xxxxxx-gcc to find its include paths. + if binary.endswith("gcc") or binary.endswith("g++"): + # (TODO): Remove 'stdin' once 'input' no longer causes issues under MSYS + result = cli.run([binary, '-E', '-Wp,-v', '-'], capture_output=True, check=True, stdin=None, input='\n') + paths = [] + for line in result.stderr.splitlines(): + if line.startswith(" "): + paths.append(Path(line.strip()).resolve()) + return paths + + return list(Path(binary).resolve().parent.parent.glob("*/include")) if binary else [] + + +@lru_cache(maxsize=10) +def cpu_defines(binary: str, compiler_args: str) -> List[str]: + cli.log.debug("gathering definitions for compilation: %s %s", binary, compiler_args) + if binary.endswith("gcc") or binary.endswith("g++"): + invocation = [binary, '-dM', '-E'] + if binary.endswith("gcc"): + invocation.extend(['-x', 'c']) + elif binary.endswith("g++"): + invocation.extend(['-x', 'c++']) + compiler_args = shlex.split(compiler_args) + invocation.extend(compiler_args) + invocation.append('-') + result = cli.run(invocation, capture_output=True, check=True, stdin=None, input='\n') + define_args = [] + for line in result.stdout.splitlines(): + line_args = line.split(' ', 2) + if len(line_args) == 3 and line_args[0] == '#define': + define_args.append(f'-D{line_args[1]}={line_args[2]}') + elif len(line_args) == 2 and line_args[0] == '#define': + define_args.append(f'-D{line_args[1]}') + return list(sorted(set(define_args))) + return [] + + +file_re = re.compile(r'printf "Compiling: ([^"]+)') +cmd_re = re.compile(r'LOG=\$\((.+?)&&') + + +def parse_make_n(f: Iterator[str]) -> List[Dict[str, str]]: + """parse the output of `make -n ` + + This function makes many assumptions about the format of your build log. + This happens to work right now for qmk. + """ + + state = 'start' + this_file = None + records = [] + for line in f: + if state == 'start': + m = file_re.search(line) + if m: + this_file = m.group(1) + state = 'cmd' + + if state == 'cmd': + assert this_file + m = cmd_re.search(line) + if m: + # we have a hit! + this_cmd = m.group(1) + args = shlex.split(this_cmd) + binary = shutil.which(args[0]) + compiler_args = set(filter(lambda x: x.startswith('-m') or x.startswith('-f'), args)) + for s in system_libs(binary): + args += ['-isystem', '%s' % s] + args.extend(cpu_defines(binary, ' '.join(shlex.quote(s) for s in compiler_args))) + new_cmd = ' '.join(shlex.quote(s) for s in args) + records.append({"directory": str(QMK_FIRMWARE.resolve()), "command": new_cmd, "file": this_file}) + state = 'start' + + return records + + +def write_compilation_database(keyboard: str = None, keymap: str = None, output_path: Path = QMK_FIRMWARE / 'compile_commands.json', skip_clean: bool = False, command: List[str] = None, **env_vars) -> bool: + # Generate the make command for a specific keyboard/keymap. + if not command: + from qmk.build_targets import KeyboardKeymapBuildTarget # Lazy load due to circular references + target = KeyboardKeymapBuildTarget(keyboard, keymap) + command = target.compile_command(dry_run=True, **env_vars) + + if not command: + cli.log.error('You must supply both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.') + cli.echo('usage: qmk generate-compilation-database [-kb KEYBOARD] [-km KEYMAP]') + return False + + # remove any environment variable overrides which could trip us up + env = os.environ.copy() + env.pop("MAKEFLAGS", None) + + # re-use same executable as the main make invocation (might be gmake) + if not skip_clean: + clean_command = [find_make(), "clean"] + cli.log.info('Making clean with {fg_cyan}%s', ' '.join(clean_command)) + cli.run(clean_command, capture_output=False, check=True, env=env) + + cli.log.info('Gathering build instructions from {fg_cyan}%s', ' '.join(command)) + + result = cli.run(command, capture_output=True, check=True, env=env) + db = parse_make_n(result.stdout.splitlines()) + if not db: + cli.log.error("Failed to parse output from make output:\n%s", result.stdout) + return False + + cli.log.info("Found %s compile commands", len(db)) + + cli.log.info(f"Writing build database to {output_path}") + output_path.write_text(json.dumps(db, indent=4)) + + return True