diff options
Diffstat (limited to 'lib/python')
| -rw-r--r-- | lib/python/qmk/build_targets.py | 211 | ||||
| -rw-r--r-- | lib/python/qmk/cli/clean.py | 4 | ||||
| -rwxr-xr-x | lib/python/qmk/cli/compile.py | 58 | ||||
| -rw-r--r-- | lib/python/qmk/cli/find.py | 14 | ||||
| -rw-r--r-- | lib/python/qmk/cli/flash.py | 45 | ||||
| -rwxr-xr-x | lib/python/qmk/cli/generate/compilation_database.py | 16 | ||||
| -rwxr-xr-x | lib/python/qmk/cli/mass_compile.py | 29 | ||||
| -rw-r--r-- | lib/python/qmk/commands.py | 164 | ||||
| -rw-r--r-- | lib/python/qmk/keyboard.py | 6 | ||||
| -rw-r--r-- | lib/python/qmk/search.py | 34 |
10 files changed, 296 insertions, 285 deletions
diff --git a/lib/python/qmk/build_targets.py b/lib/python/qmk/build_targets.py new file mode 100644 index 0000000000..236c2eaa03 --- /dev/null +++ b/lib/python/qmk/build_targets.py @@ -0,0 +1,211 @@ +# Copyright 2023 Nick Brassel (@tzarc) +# SPDX-License-Identifier: GPL-2.0-or-later +import json +import shutil +from typing import List, Union +from pathlib import Path +from dotty_dict import dotty, Dotty +from milc import cli +from qmk.constants import QMK_FIRMWARE, INTERMEDIATE_OUTPUT_PREFIX +from qmk.commands import find_make, get_make_parallel_args, parse_configurator_json +from qmk.keyboard import keyboard_folder +from qmk.info import keymap_json +from qmk.cli.generate.compilation_database import write_compilation_database + + +class BuildTarget: + def __init__(self, keyboard: str, keymap: str, json: Union[dict, Dotty] = None): + self._keyboard = keyboard_folder(keyboard) + self._keyboard_safe = self._keyboard.replace('/', '_') + self._keymap = keymap + self._parallel = 1 + self._clean = False + self._compiledb = False + self._target = f'{self._keyboard_safe}_{self.keymap}' + self._intermediate_output = Path(f'{INTERMEDIATE_OUTPUT_PREFIX}{self._target}') + self._generated_files_path = self._intermediate_output / 'src' + self._json = json.to_dict() if isinstance(json, Dotty) else json + + def __str__(self): + return f'{self.keyboard}:{self.keymap}' + + def __repr__(self): + return f'BuildTarget(keyboard={self.keyboard}, keymap={self.keymap})' + + def configure(self, parallel: int = None, clean: bool = None, compiledb: bool = None) -> None: + if parallel is not None: + self._parallel = parallel + if clean is not None: + self._clean = clean + if compiledb is not None: + self._compiledb = compiledb + + @property + def keyboard(self) -> str: + return self._keyboard + + @property + def keymap(self) -> str: + return self._keymap + + @property + def json(self) -> dict: + if not self._json: + self._load_json() + if not self._json: + return {} + return self._json + + @property + def dotty(self) -> Dotty: + return dotty(self.json) + + def _common_make_args(self, dry_run: bool = False, build_target: str = None): + compile_args = [ + find_make(), + *get_make_parallel_args(self._parallel), + '-r', + '-R', + '-f', + 'builddefs/build_keyboard.mk', + ] + + if not cli.config.general.verbose: + compile_args.append('-s') + + verbose = 'true' if cli.config.general.verbose else 'false' + color = 'true' if cli.config.general.color else 'false' + + if dry_run: + compile_args.append('-n') + + if build_target: + compile_args.append(build_target) + + compile_args.extend([ + f'KEYBOARD={self.keyboard}', + f'KEYMAP={self.keymap}', + f'KEYBOARD_FILESAFE={self._keyboard_safe}', + f'TARGET={self._target}', + f'INTERMEDIATE_OUTPUT={self._intermediate_output}', + f'VERBOSE={verbose}', + f'COLOR={color}', + 'SILENT=false', + 'QMK_BIN="qmk"', + ]) + + return compile_args + + def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None: + raise NotImplementedError("prepare_build() not implemented in base class") + + def compile_command(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]: + raise NotImplementedError("compile_command() not implemented in base class") + + def generate_compilation_database(self, build_target: str = None, skip_clean: bool = False, **env_vars) -> None: + self.prepare_build(build_target=build_target, **env_vars) + command = self.compile_command(build_target=build_target, dry_run=True, **env_vars) + write_compilation_database(command=command, output_path=QMK_FIRMWARE / 'compile_commands.json', skip_clean=skip_clean, **env_vars) + + def compile(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None: + if self._clean or self._compiledb: + command = [find_make(), "clean"] + if dry_run: + command.append('-n') + cli.log.info('Cleaning with {fg_cyan}%s', ' '.join(command)) + cli.run(command, capture_output=False) + + if self._compiledb and not dry_run: + self.generate_compilation_database(build_target=build_target, skip_clean=True, **env_vars) + + self.prepare_build(build_target=build_target, dry_run=dry_run, **env_vars) + command = self.compile_command(build_target=build_target, **env_vars) + cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command)) + if not dry_run: + cli.echo('\n') + ret = cli.run(command, capture_output=False) + if ret.returncode: + return ret.returncode + + +class KeyboardKeymapBuildTarget(BuildTarget): + def __init__(self, keyboard: str, keymap: str, json: dict = None): + super().__init__(keyboard=keyboard, keymap=keymap, json=json) + + def __repr__(self): + return f'KeyboardKeymapTarget(keyboard={self.keyboard}, keymap={self.keymap})' + + def _load_json(self): + self._json = keymap_json(self.keyboard, self.keymap) + + def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None: + pass + + def compile_command(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]: + compile_args = self._common_make_args(dry_run=dry_run, build_target=build_target) + + for key, value in env_vars.items(): + compile_args.append(f'{key}={value}') + + return compile_args + + +class JsonKeymapBuildTarget(BuildTarget): + def __init__(self, json_path): + if isinstance(json_path, Path): + self.json_path = json_path + else: + self.json_path = None + + json = parse_configurator_json(json_path) # Will load from stdin if provided + + # In case the user passes a keymap.json from a keymap directory directly to the CLI. + # e.g.: qmk compile - < keyboards/clueboard/california/keymaps/default/keymap.json + json["keymap"] = json.get("keymap", "default_json") + + super().__init__(keyboard=json['keyboard'], keymap=json['keymap'], json=json) + + self._keymap_json = self._generated_files_path / 'keymap.json' + + def __repr__(self): + return f'JsonKeymapTarget(keyboard={self.keyboard}, keymap={self.keymap}, path={self.json_path})' + + def _load_json(self): + pass # Already loaded in constructor + + def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None: + if self._clean: + if self._intermediate_output.exists(): + shutil.rmtree(self._intermediate_output) + + # begin with making the deepest folder in the tree + self._generated_files_path.mkdir(exist_ok=True, parents=True) + + # Compare minified to ensure consistent comparison + new_content = json.dumps(self.json, separators=(',', ':')) + if self._keymap_json.exists(): + old_content = json.dumps(json.loads(self._keymap_json.read_text(encoding='utf-8')), separators=(',', ':')) + if old_content == new_content: + new_content = None + + # Write the keymap.json file if different so timestamps are only updated + # if the content changes -- running `make` won't treat it as modified. + if new_content: + self._keymap_json.write_text(new_content, encoding='utf-8') + + def compile_command(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]: + compile_args = self._common_make_args(dry_run=dry_run, build_target=build_target) + compile_args.extend([ + f'MAIN_KEYMAP_PATH_1={self._intermediate_output}', + f'MAIN_KEYMAP_PATH_2={self._intermediate_output}', + f'MAIN_KEYMAP_PATH_3={self._intermediate_output}', + f'MAIN_KEYMAP_PATH_4={self._intermediate_output}', + f'MAIN_KEYMAP_PATH_5={self._intermediate_output}', + f'KEYMAP_JSON={self._keymap_json}', + f'KEYMAP_PATH={self._generated_files_path}', + ]) + + for key, value in env_vars.items(): + compile_args.append(f'{key}={value}') + + return compile_args diff --git a/lib/python/qmk/cli/clean.py b/lib/python/qmk/cli/clean.py index 72b7ffe810..bdec01e4b6 100644 --- a/lib/python/qmk/cli/clean.py +++ b/lib/python/qmk/cli/clean.py @@ -2,7 +2,7 @@ """ from subprocess import DEVNULL -from qmk.commands import create_make_target +from qmk.commands import find_make from milc import cli @@ -11,4 +11,4 @@ from milc import cli def clean(cli): """Runs `make clean` (or `make distclean` if --all is passed) """ - cli.run(create_make_target('distclean' if cli.args.all else 'clean'), capture_output=False, stdin=DEVNULL) + cli.run([find_make(), 'distclean' if cli.args.all else 'clean'], capture_output=False, stdin=DEVNULL) diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py index 2478ea1fed..71c1dec162 100755 --- a/lib/python/qmk/cli/compile.py +++ b/lib/python/qmk/cli/compile.py @@ -7,22 +7,11 @@ from argcomplete.completers import FilesCompleter from milc import cli import qmk.path -from qmk.constants import QMK_FIRMWARE from qmk.decorators import automagic_keyboard, automagic_keymap -from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json, build_environment +from qmk.commands import build_environment from qmk.keyboard import keyboard_completer, keyboard_folder_or_all, is_all_keyboards from qmk.keymap import keymap_completer, locate_keymap -from qmk.cli.generate.compilation_database import write_compilation_database - - -def _is_keymap_target(keyboard, keymap): - if keymap == 'all': - return True - - if locate_keymap(keyboard, keymap): - return True - - return False +from qmk.build_targets import KeyboardKeymapBuildTarget, JsonKeymapBuildTarget @cli.argument('filename', nargs='?', arg_only=True, type=qmk.path.FileType('r'), completer=FilesCompleter('.json'), help='The configurator export to compile') @@ -32,6 +21,7 @@ def _is_keymap_target(keyboard, keymap): @cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.") @cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.") @cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.") +@cli.argument('-t', '--target', type=str, default=None, help="Intended alternative build target, such as `production` in `make planck/rev4:default:production`.") @cli.argument('--compiledb', arg_only=True, action='store_true', help="Generates the clang compile_commands.json file during build. Implies --clean.") @cli.subcommand('Compile a QMK Firmware.') @automagic_keyboard @@ -53,47 +43,27 @@ def compile(cli): # Build the environment vars envs = build_environment(cli.args.env) - # Determine the compile command - commands = [] - - current_keyboard = None - current_keymap = None + # Handler for the build target + target = None if cli.args.filename: - # If a configurator JSON was provided generate a keymap and compile it - user_keymap = parse_configurator_json(cli.args.filename) - commands = [compile_configurator_json(user_keymap, parallel=cli.config.compile.parallel, clean=cli.args.clean, **envs)] + # if we were given a filename, assume we have a json build target + target = JsonKeymapBuildTarget(cli.args.filename) elif cli.config.compile.keyboard and cli.config.compile.keymap: - # Generate the make command for a specific keyboard/keymap. - if not _is_keymap_target(cli.config.compile.keyboard, cli.config.compile.keymap): + # if we got a keyboard and keymap, attempt to find it + if not locate_keymap(cli.config.compile.keyboard, cli.config.compile.keymap): cli.log.error('Invalid keymap argument.') cli.print_help() return False - if cli.args.clean: - commands.append(create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, 'clean', **envs)) - commands.append(create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, parallel=cli.config.compile.parallel, **envs)) + # If we got here, then we have a valid keyboard and keymap for a build target + target = KeyboardKeymapBuildTarget(cli.config.compile.keyboard, cli.config.compile.keymap) - current_keyboard = cli.config.compile.keyboard - current_keymap = cli.config.compile.keymap - - if not commands: + if not target: cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.') cli.print_help() return False - if cli.args.compiledb: - if current_keyboard is None or current_keymap is None: - cli.log.error('You must supply both `--keyboard` and `--keymap` or be in a directory with a keymap to generate a compile_commands.json file.') - cli.print_help() - return False - write_compilation_database(current_keyboard, current_keymap, QMK_FIRMWARE / 'compile_commands.json') - - cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(commands[-1])) - if not cli.args.dry_run: - cli.echo('\n') - for command in commands: - ret = cli.run(command, capture_output=False) - if ret.returncode: - return ret.returncode + target.configure(parallel=cli.config.compile.parallel, clean=cli.args.clean, compiledb=cli.args.compiledb) + target.compile(cli.args.target, dry_run=cli.args.dry_run, **envs) diff --git a/lib/python/qmk/cli/find.py b/lib/python/qmk/cli/find.py index f2135bbc16..55a0530092 100644 --- a/lib/python/qmk/cli/find.py +++ b/lib/python/qmk/cli/find.py @@ -19,13 +19,9 @@ from qmk.search import search_keymap_targets def find(cli): """Search through all keyboards and keymaps for a given search criteria. """ + targets = search_keymap_targets([('all', cli.config.find.keymap)], cli.args.filter) + for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)): + print(f'{target}') - if len(cli.args.filter) == 0 and len(cli.args.print) > 0: - cli.log.warning('No filters supplied -- keymaps not parsed, unable to print requested values.') - - targets = search_keymap_targets([('all', cli.config.find.keymap)], cli.args.filter, cli.args.print) - for keyboard, keymap, print_vals in targets: - print(f'{keyboard}:{keymap}') - - for key, val in print_vals: - print(f' {key}={val}') + for key in cli.args.print: + print(f' {key}={target.dotty.get(key, None)}') diff --git a/lib/python/qmk/cli/flash.py b/lib/python/qmk/cli/flash.py index 8724f26889..5df31e4781 100644 --- a/lib/python/qmk/cli/flash.py +++ b/lib/python/qmk/cli/flash.py @@ -4,25 +4,17 @@ You can compile a keymap already in the repo or using a QMK Configurator export. A bootloader must be specified. """ from argcomplete.completers import FilesCompleter +from pathlib import Path from milc import cli import qmk.path from qmk.decorators import automagic_keyboard, automagic_keymap -from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json, build_environment +from qmk.commands import build_environment from qmk.keyboard import keyboard_completer, keyboard_folder from qmk.keymap import keymap_completer, locate_keymap from qmk.flashers import flasher - - -def _is_keymap_target(keyboard, keymap): - if keymap == 'all': - return True - - if locate_keymap(keyboard, keymap): - return True - - return False +from qmk.build_targets import KeyboardKeymapBuildTarget, JsonKeymapBuildTarget def _list_bootloaders(): @@ -89,7 +81,7 @@ def flash(cli): If bootloader is omitted the make system will use the configured bootloader for that keyboard. """ - if cli.args.filename and cli.args.filename.suffix in ['.bin', '.hex', '.uf2']: + if cli.args.filename and isinstance(cli.args.filename, Path) and cli.args.filename.suffix in ['.bin', '.hex', '.uf2']: return _flash_binary(cli.args.filename, cli.args.mcu) if cli.args.bootloaders: @@ -98,34 +90,27 @@ def flash(cli): # Build the environment vars envs = build_environment(cli.args.env) - # Determine the compile command - commands = [] + # Handler for the build target + target = None if cli.args.filename: - # If a configurator JSON was provided generate a keymap and compile it - user_keymap = parse_configurator_json(cli.args.filename) - commands = [compile_configurator_json(user_keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, clean=cli.args.clean, **envs)] + # if we were given a filename, assume we have a json build target + target = JsonKeymapBuildTarget(cli.args.filename) elif cli.config.flash.keyboard and cli.config.flash.keymap: - # Generate the make command for a specific keyboard/keymap. - if not _is_keymap_target(cli.config.flash.keyboard, cli.config.flash.keymap): + # if we got a keyboard and keymap, attempt to find it + if not locate_keymap(cli.config.flash.keyboard, cli.config.flash.keymap): cli.log.error('Invalid keymap argument.') cli.print_help() return False - if cli.args.clean: - commands.append(create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, 'clean', **envs)) - commands.append(create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, **envs)) + # If we got here, then we have a valid keyboard and keymap for a build target + target = KeyboardKeymapBuildTarget(cli.config.flash.keyboard, cli.config.flash.keymap) - if not commands: + if not target: cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.') cli.print_help() return False - cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(commands[-1])) - if not cli.args.dry_run: - cli.echo('\n') - for command in commands: - ret = cli.run(command, capture_output=False) - if ret.returncode: - return ret.returncode + target.configure(parallel=cli.config.flash.parallel, clean=cli.args.clean) + target.compile(cli.args.bootloader, dry_run=cli.args.dry_run, **envs) diff --git a/lib/python/qmk/cli/generate/compilation_database.py b/lib/python/qmk/cli/generate/compilation_database.py index 9e5c266516..5100d2b6d2 100755 --- a/lib/python/qmk/cli/generate/compilation_database.py +++ b/lib/python/qmk/cli/generate/compilation_database.py @@ -12,7 +12,7 @@ from typing import Dict, Iterator, List, Union from milc import cli, MILC -from qmk.commands import create_make_command +from qmk.commands import find_make from qmk.constants import QMK_FIRMWARE from qmk.decorators import automagic_keyboard, automagic_keymap from qmk.keyboard import keyboard_completer, keyboard_folder @@ -76,9 +76,12 @@ def parse_make_n(f: Iterator[str]) -> List[Dict[str, str]]: return records -def write_compilation_database(keyboard: str, keymap: str, output_path: Path) -> bool: +def write_compilation_database(keyboard: str = None, keymap: str = None, output_path: Path = QMK_FIRMWARE / 'compile_commands.json', skip_clean: bool = False, command: List[str] = None, **env_vars) -> bool: # Generate the make command for a specific keyboard/keymap. - command = create_make_command(keyboard, keymap, dry_run=True) + if not command: + from qmk.build_targets import KeyboardKeymapBuildTarget # Lazy load due to circular references + target = KeyboardKeymapBuildTarget(keyboard, keymap) + command = target.compile_command(dry_run=True, **env_vars) if not command: cli.log.error('You must supply both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.') @@ -90,9 +93,10 @@ def write_compilation_database(keyboard: str, keymap: str, output_path: Path) -> env.pop("MAKEFLAGS", None) # re-use same executable as the main make invocation (might be gmake) - clean_command = [command[0], 'clean'] - cli.log.info('Making clean with {fg_cyan}%s', ' '.join(clean_command)) - cli.run(clean_command, capture_output=False, check=True, env=env) + if not skip_clean: + clean_command = [find_make(), "clean"] + cli.log.info('Making clean with {fg_cyan}%s', ' '.join(clean_command)) + cli.run(clean_command, capture_output=False, check=True, env=env) cli.log.info('Gathering build instructions from {fg_cyan}%s', ' '.join(command)) diff --git a/lib/python/qmk/cli/mass_compile.py b/lib/python/qmk/cli/mass_compile.py index 06e6e411a7..7968de53e7 100755 --- a/lib/python/qmk/cli/mass_compile.py +++ b/lib/python/qmk/cli/mass_compile.py @@ -3,26 +3,28 @@ This will compile everything in parallel, for testing purposes. """ import os +from typing import List from pathlib import Path from subprocess import DEVNULL from milc import cli from qmk.constants import QMK_FIRMWARE -from qmk.commands import _find_make, get_make_parallel_args +from qmk.commands import find_make, get_make_parallel_args, build_environment from qmk.search import search_keymap_targets, search_make_targets +from qmk.build_targets import BuildTarget, JsonKeymapBuildTarget -def mass_compile_targets(targets, clean, dry_run, no_temp, parallel, env): +def mass_compile_targets(targets: List[BuildTarget], clean: bool, dry_run: bool, no_temp: bool, parallel: int, **env): if len(targets) == 0: return - make_cmd = _find_make() + make_cmd = find_make() builddir = Path(QMK_FIRMWARE) / '.build' makefile = builddir / 'parallel_kb_builds.mk' if dry_run: cli.log.info('Compilation targets:') - for target in sorted(targets): + for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)): cli.log.info(f"{{fg_cyan}}qmk compile -kb {target[0]} -km {target[1]}{{fg_reset}}") else: if clean: @@ -30,9 +32,13 @@ def mass_compile_targets(targets, clean, dry_run, no_temp, parallel, env): builddir.mkdir(parents=True, exist_ok=True) with open(makefile, "w") as f: - for target in sorted(targets): - keyboard_name = target[0] - keymap_name = target[1] + for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)): + keyboard_name = target.keyboard + keymap_name = target.keymap + target.configure(parallel=1) # We ignore parallelism on a per-build basis as we defer to the parent make invocation + target.prepare_build(**env) # If we've got json targets, allow them to write out any extra info to .build before we kick off `make` + command = target.compile_command(**env) + command[0] = '+@$(MAKE)' # Override the make so that we can use jobserver to handle parallelism keyboard_safe = keyboard_name.replace('/', '_') build_log = f"{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}.{keymap_name}" failed_log = f"{QMK_FIRMWARE}/.build/failed.log.{os.getpid()}.{keyboard_safe}.{keymap_name}" @@ -43,7 +49,7 @@ all: {keyboard_safe}_{keymap_name}_binary {keyboard_safe}_{keymap_name}_binary: @rm -f "{build_log}" || true @echo "Compiling QMK Firmware for target: '{keyboard_name}:{keymap_name}'..." >>"{build_log}" - +@$(MAKE) -C "{QMK_FIRMWARE}" -f "{QMK_FIRMWARE}/builddefs/build_keyboard.mk" KEYBOARD="{keyboard_name}" KEYMAP="{keymap_name}" COLOR=true SILENT=false {' '.join(env)} \\ + {' '.join(command)} \\ >>"{build_log}" 2>&1 \\ || cp "{build_log}" "{failed_log}" @{{ grep '\[ERRORS\]' "{build_log}" >/dev/null 2>&1 && printf "Build %-64s \e[1;31m[ERRORS]\e[0m\\n" "{keyboard_name}:{keymap_name}" ; }} \\ @@ -95,8 +101,11 @@ def mass_compile(cli): """Compile QMK Firmware against all keyboards. """ if len(cli.args.builds) > 0: - targets = search_make_targets(cli.args.builds, cli.args.filter) + json_like_targets = list([Path(p) for p in filter(lambda e: Path(e).exists() and Path(e).suffix == '.json', cli.args.builds)]) + make_like_targets = list(filter(lambda e: Path(e) not in json_like_targets, cli.args.builds)) + targets = search_make_targets(make_like_targets) + targets.extend([JsonKeymapBuildTarget(e) for e in json_like_targets]) else: targets = search_keymap_targets([('all', cli.config.mass_compile.keymap)], cli.args.filter) - return mass_compile_targets(targets, cli.args.clean, cli.args.dry_run, cli.config.mass_compile.no_temp, cli.config.mass_compile.parallel, cli.args.env) + return mass_compile_targets(targets, cli.args.clean, cli.args.dry_run, cli.config.mass_compile.no_temp, cli.config.mass_compile.parallel, **build_environment(cli.args.env)) diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py index 34696e3793..e5fdbc4f30 100644 --- a/lib/python/qmk/commands.py +++ b/lib/python/qmk/commands.py @@ -2,18 +2,16 @@ """ import os import sys -import json import shutil from pathlib import Path from milc import cli import jsonschema -from qmk.constants import INTERMEDIATE_OUTPUT_PREFIX from qmk.json_schema import json_load, validate -def _find_make(): +def find_make(): """Returns the correct make command for this environment. """ make_cmd = os.environ.get('MAKE') @@ -24,74 +22,6 @@ def _find_make(): return make_cmd -def create_make_target(target, dry_run=False, parallel=1, **env_vars): - """Create a make command - - Args: - - target - Usually a make rule, such as 'clean' or 'all'. - - dry_run - make -n -- don't actually build - - parallel - The number of make jobs to run in parallel - - **env_vars - Environment variables to be passed to make. - - Returns: - - A command that can be run to make the specified keyboard and keymap - """ - env = [] - make_cmd = _find_make() - - for key, value in env_vars.items(): - env.append(f'{key}={value}') - - if cli.config.general.verbose: - env.append('VERBOSE=true') - - return [make_cmd, *(['-n'] if dry_run else []), *get_make_parallel_args(parallel), *env, target] - - -def create_make_command(keyboard, keymap, target=None, dry_run=False, parallel=1, **env_vars): - """Create a make compile command - - Args: - - keyboard - The path of the keyboard, for example 'plank' - - keymap - The name of the keymap, for example 'algernon' - - target - Usually a bootloader. - - dry_run - make -n -- don't actually build - - parallel - The number of make jobs to run in parallel - - **env_vars - Environment variables to be passed to make. - - Returns: - - A command that can be run to make the specified keyboard and keymap - """ - make_args = [keyboard, keymap] - - if target: - make_args.append(target) - - return create_make_target(':'.join(make_args), dry_run=dry_run, parallel=parallel, **env_vars) - - def get_make_parallel_args(parallel=1): """Returns the arguments for running the specified number of parallel jobs. """ @@ -100,7 +30,7 @@ def get_make_parallel_args(parallel=1): if int(parallel) <= 0: # 0 or -1 means -j without argument (unlimited jobs) parallel_args.append('--jobs') - else: + elif int(parallel) > 1: parallel_args.append('--jobs=' + str(parallel)) if int(parallel) != 1: @@ -110,96 +40,6 @@ def get_make_parallel_args(parallel=1): return parallel_args -def compile_configurator_json(user_keymap, bootloader=None, parallel=1, clean=False, **env_vars): - """Convert a configurator export JSON file into a C file and then compile it. - - Args: - - user_keymap - A deserialized keymap export - - bootloader - A bootloader to flash - - parallel - The number of make jobs to run in parallel - - Returns: - - A command to run to compile and flash the C file. - """ - # In case the user passes a keymap.json from a keymap directory directly to the CLI. - # e.g.: qmk compile - < keyboards/clueboard/california/keymaps/default/keymap.json - user_keymap["keymap"] = user_keymap.get("keymap", "default_json") - - keyboard_filesafe = user_keymap['keyboard'].replace('/', '_') - target = f'{keyboard_filesafe}_{user_keymap["keymap"]}' - intermediate_output = Path(f'{INTERMEDIATE_OUTPUT_PREFIX}{keyboard_filesafe}_{user_keymap["keymap"]}') - keymap_dir = intermediate_output / 'src' - keymap_json = keymap_dir / 'keymap.json' - - if clean: - if intermediate_output.exists(): - shutil.rmtree(intermediate_output) - - # begin with making the deepest folder in the tree - keymap_dir.mkdir(exist_ok=True, parents=True) - - # Compare minified to ensure consistent comparison - new_content = json.dumps(user_keymap, separators=(',', ':')) - if keymap_json.exists(): - old_content = json.dumps(json.loads(keymap_json.read_text(encoding='utf-8')), separators=(',', ':')) - if old_content == new_content: - new_content = None - - # Write the keymap.json file if different - if new_content: - keymap_json.write_text(new_content, encoding='utf-8') - - # Return a command that can be run to make the keymap and flash if given - verbose = 'true' if cli.config.general.verbose else 'false' - color = 'true' if cli.config.general.color else 'false' - make_command = [_find_make()] - - if not cli.config.general.verbose: - make_command.append('-s') - - make_command.extend([ - *get_make_parallel_args(parallel), - '-r', - '-R', - '-f', - 'builddefs/build_keyboard.mk', - ]) - - if bootloader: - make_command.append(bootloader) - - make_command.extend([ - f'KEYBOARD={user_keymap["keyboard"]}', - f'KEYMAP={user_keymap["keymap"]}', - f'KEYBOARD_FILESAFE={keyboard_filesafe}', - f'TARGET={target}', - f'INTERMEDIATE_OUTPUT={intermediate_output}', - f'MAIN_KEYMAP_PATH_1={intermediate_output}', - f'MAIN_KEYMAP_PATH_2={intermediate_output}', - f'MAIN_KEYMAP_PATH_3={intermediate_output}', - f'MAIN_KEYMAP_PATH_4={intermediate_output}', - f'MAIN_KEYMAP_PATH_5={intermediate_output}', - f'KEYMAP_JSON={keymap_json}', - f'KEYMAP_PATH={keymap_dir}', - f'VERBOSE={verbose}', - f'COLOR={color}', - 'SILENT=false', - 'QMK_BIN="qmk"', - ]) - - for key, value in env_vars.items(): - make_command.append(f'{key}={value}') - - return make_command - - def parse_configurator_json(configurator_file): """Open and parse a configurator json export """ diff --git a/lib/python/qmk/keyboard.py b/lib/python/qmk/keyboard.py index 9826f3f887..1aa63687d4 100644 --- a/lib/python/qmk/keyboard.py +++ b/lib/python/qmk/keyboard.py @@ -98,11 +98,7 @@ def keyboard_folder(keyboard): if keyboard == last_keyboard: break - rules_mk_file = Path(base_path, keyboard, 'rules.mk') - - if rules_mk_file.exists(): - rules_mk = parse_rules_mk_file(rules_mk_file) - keyboard = rules_mk.get('DEFAULT_FOLDER', keyboard) + keyboard = resolve_keyboard(keyboard) if not qmk.path.is_keyboard(keyboard): raise ValueError(f'Invalid keyboard: {keyboard}') diff --git a/lib/python/qmk/search.py b/lib/python/qmk/search.py index a74450ca87..301d69c6c2 100644 --- a/lib/python/qmk/search.py +++ b/lib/python/qmk/search.py @@ -11,8 +11,9 @@ from milc import cli from qmk.util import parallel_map from qmk.info import keymap_json -import qmk.keyboard -import qmk.keymap +from qmk.keyboard import list_keyboards, keyboard_folder +from qmk.keymap import list_keymaps, locate_keymap +from qmk.build_targets import KeyboardKeymapBuildTarget, BuildTarget def _set_log_level(level): @@ -36,15 +37,15 @@ def _all_keymaps(keyboard): """Returns a list of tuples of (keyboard, keymap) for all keymaps for the given keyboard. """ with ignore_logging(): - keyboard = qmk.keyboard.resolve_keyboard(keyboard) - return [(keyboard, keymap) for keymap in qmk.keymap.list_keymaps(keyboard)] + keyboard = keyboard_folder(keyboard) + return [(keyboard, keymap) for keymap in list_keymaps(keyboard)] def _keymap_exists(keyboard, keymap): """Returns the keyboard name if the keyboard+keymap combination exists, otherwise None. """ with ignore_logging(): - return keyboard if qmk.keymap.locate_keymap(keyboard, keymap) is not None else None + return keyboard if locate_keymap(keyboard, keymap) is not None else None def _load_keymap_info(kb_km): @@ -75,7 +76,7 @@ def _expand_keymap_target(keyboard: str, keymap: str, all_keyboards: List[str] = Caters for 'all' in either keyboard or keymap, or both. """ if all_keyboards is None: - all_keyboards = qmk.keyboard.list_keyboards() + all_keyboards = list_keyboards() if keyboard == 'all': if keymap == 'all': @@ -90,30 +91,29 @@ def _expand_keymap_target(keyboard: str, keymap: str, all_keyboards: List[str] = return [(kb, keymap) for kb in filter(lambda e: e is not None, parallel_map(keyboard_filter, all_keyboards))] else: if keymap == 'all': - keyboard = qmk.keyboard.resolve_keyboard(keyboard) cli.log.info(f'Retrieving list of keymaps for keyboard "{keyboard}"...') return _all_keymaps(keyboard) else: - return [(qmk.keyboard.resolve_keyboard(keyboard), keymap)] + return [(keyboard, keymap)] def expand_keymap_targets(targets: List[Tuple[str, str]]) -> List[Tuple[str, str]]: """Expand a list of (keyboard, keymap) tuples inclusive of 'all', into a list of explicit (keyboard, keymap) tuples. """ overall_targets = [] - all_keyboards = qmk.keyboard.list_keyboards() + all_keyboards = list_keyboards() for target in targets: overall_targets.extend(_expand_keymap_target(target[0], target[1], all_keyboards)) return list(sorted(set(overall_targets))) -def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str] = [], print_vals: List[str] = []) -> List[Tuple[str, str, List[Tuple[str, str]]]]: +def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str] = []) -> List[BuildTarget]: """Filter a list of (keyboard, keymap) tuples based on the supplied filters. Optionally includes the values of the queried info.json keys. """ - if len(filters) == 0 and len(print_vals) == 0: - targets = [(kb, km, {}) for kb, km in target_list] + if len(filters) == 0: + targets = [KeyboardKeymapBuildTarget(keyboard=kb, keymap=km) for kb, km in target_list] else: cli.log.info('Parsing data for all matching keyboard/keymap combinations...') valid_keymaps = [(e[0], e[1], dotty(e[2])) for e in parallel_map(_load_keymap_info, target_list)] @@ -172,18 +172,18 @@ def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str cli.log.warning(f'Unrecognized filter expression: {filter_expr}') continue - targets = [(e[0], e[1], [(p, e[2].get(p)) for p in print_vals]) for e in valid_keymaps] + targets = [KeyboardKeymapBuildTarget(keyboard=e[0], keymap=e[1], json=e[2]) for e in valid_keymaps] return targets -def search_keymap_targets(targets: List[Tuple[str, str]] = [('all', 'default')], filters: List[str] = [], print_vals: List[str] = []) -> List[Tuple[str, str, List[Tuple[str, str]]]]: +def search_keymap_targets(targets: List[Tuple[str, str]] = [('all', 'default')], filters: List[str] = []) -> List[BuildTarget]: """Search for build targets matching the supplied criteria. """ - return list(sorted(_filter_keymap_targets(expand_keymap_targets(targets), filters, print_vals), key=lambda e: (e[0], e[1]))) + return _filter_keymap_targets(expand_keymap_targets(targets), filters) -def search_make_targets(targets: List[str], filters: List[str] = [], print_vals: List[str] = []) -> List[Tuple[str, str, List[Tuple[str, str]]]]: +def search_make_targets(targets: List[str], filters: List[str] = []) -> List[BuildTarget]: """Search for build targets matching the supplied criteria. """ - return list(sorted(_filter_keymap_targets(expand_make_targets(targets), filters, print_vals), key=lambda e: (e[0], e[1]))) + return _filter_keymap_targets(expand_make_targets(targets), filters) |