diff options
Diffstat (limited to 'lib/python')
| -rw-r--r-- | lib/python/qmk/build_targets.py | 16 | ||||
| -rw-r--r-- | lib/python/qmk/cli/__init__.py | 5 | ||||
| -rwxr-xr-x | lib/python/qmk/cli/compile.py | 4 | ||||
| -rwxr-xr-x | lib/python/qmk/cli/doctor/main.py | 25 | ||||
| -rwxr-xr-x | lib/python/qmk/cli/format/json.py | 70 | ||||
| -rwxr-xr-x | lib/python/qmk/cli/mass_compile.py | 2 | ||||
| -rwxr-xr-x | lib/python/qmk/cli/new/keymap.py | 8 | ||||
| -rw-r--r-- | lib/python/qmk/cli/userspace/__init__.py | 5 | ||||
| -rw-r--r-- | lib/python/qmk/cli/userspace/add.py | 51 | ||||
| -rw-r--r-- | lib/python/qmk/cli/userspace/compile.py | 38 | ||||
| -rw-r--r-- | lib/python/qmk/cli/userspace/doctor.py | 11 | ||||
| -rw-r--r-- | lib/python/qmk/cli/userspace/list.py | 51 | ||||
| -rw-r--r-- | lib/python/qmk/cli/userspace/remove.py | 37 | ||||
| -rw-r--r-- | lib/python/qmk/commands.py | 6 | ||||
| -rw-r--r-- | lib/python/qmk/constants.py | 8 | ||||
| -rwxr-xr-x | lib/python/qmk/json_encoders.py | 18 | ||||
| -rw-r--r-- | lib/python/qmk/keyboard.py | 22 | ||||
| -rw-r--r-- | lib/python/qmk/keymap.py | 130 | ||||
| -rw-r--r-- | lib/python/qmk/path.py | 59 | ||||
| -rw-r--r-- | lib/python/qmk/userspace.py | 185 |
20 files changed, 676 insertions, 75 deletions
diff --git a/lib/python/qmk/build_targets.py b/lib/python/qmk/build_targets.py index 16a7ef87a2..1ab489cec3 100644 --- a/lib/python/qmk/build_targets.py +++ b/lib/python/qmk/build_targets.py @@ -10,6 +10,8 @@ from qmk.constants import QMK_FIRMWARE, INTERMEDIATE_OUTPUT_PREFIX from qmk.commands import find_make, get_make_parallel_args, parse_configurator_json from qmk.keyboard import keyboard_folder from qmk.info import keymap_json +from qmk.keymap import locate_keymap +from qmk.path import is_under_qmk_firmware, is_under_qmk_userspace class BuildTarget: @@ -158,6 +160,20 @@ class KeyboardKeymapBuildTarget(BuildTarget): for key, value in env_vars.items(): compile_args.append(f'{key}={value}') + # Need to override the keymap path if the keymap is a userspace directory. + # This also ensures keyboard aliases as per `keyboard_aliases.hjson` still work if the userspace has the keymap + # in an equivalent historical location. + keymap_location = locate_keymap(self.keyboard, self.keymap) + if is_under_qmk_userspace(keymap_location) and not is_under_qmk_firmware(keymap_location): + keymap_directory = keymap_location.parent + compile_args.extend([ + f'MAIN_KEYMAP_PATH_1={keymap_directory}', + f'MAIN_KEYMAP_PATH_2={keymap_directory}', + f'MAIN_KEYMAP_PATH_3={keymap_directory}', + f'MAIN_KEYMAP_PATH_4={keymap_directory}', + f'MAIN_KEYMAP_PATH_5={keymap_directory}', + ]) + return compile_args diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py index 695a180066..cf60903687 100644 --- a/lib/python/qmk/cli/__init__.py +++ b/lib/python/qmk/cli/__init__.py @@ -81,6 +81,11 @@ subcommands = [ 'qmk.cli.new.keymap', 'qmk.cli.painter', 'qmk.cli.pytest', + 'qmk.cli.userspace.add', + 'qmk.cli.userspace.compile', + 'qmk.cli.userspace.doctor', + 'qmk.cli.userspace.list', + 'qmk.cli.userspace.remove', 'qmk.cli.via2json', ] diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py index 71c1dec162..3c8f3664ea 100755 --- a/lib/python/qmk/cli/compile.py +++ b/lib/python/qmk/cli/compile.py @@ -37,7 +37,9 @@ def compile(cli): from .mass_compile import mass_compile cli.args.builds = [] cli.args.filter = [] - cli.args.no_temp = False + cli.config.mass_compile.keymap = cli.config.compile.keymap + cli.config.mass_compile.parallel = cli.config.compile.parallel + cli.config.mass_compile.no_temp = False return mass_compile(cli) # Build the environment vars diff --git a/lib/python/qmk/cli/doctor/main.py b/lib/python/qmk/cli/doctor/main.py index 6a6feb87d1..dd8b58b2c7 100755 --- a/lib/python/qmk/cli/doctor/main.py +++ b/lib/python/qmk/cli/doctor/main.py @@ -9,10 +9,11 @@ from milc import cli from milc.questions import yesno from qmk import submodules -from qmk.constants import QMK_FIRMWARE, QMK_FIRMWARE_UPSTREAM +from qmk.constants import QMK_FIRMWARE, QMK_FIRMWARE_UPSTREAM, QMK_USERSPACE, HAS_QMK_USERSPACE from .check import CheckStatus, check_binaries, check_binary_versions, check_submodules from qmk.git import git_check_repo, git_get_branch, git_get_tag, git_get_last_log_entry, git_get_common_ancestor, git_is_dirty, git_get_remotes, git_check_deviation from qmk.commands import in_virtualenv +from qmk.userspace import qmk_userspace_paths, qmk_userspace_validate, UserspaceValidationError def os_tests(): @@ -92,6 +93,25 @@ def output_submodule_status(): cli.log.error(f'- {sub_name}: <<< missing or unknown >>>') +def userspace_tests(qmk_firmware): + if qmk_firmware: + cli.log.info(f'QMK home: {{fg_cyan}}{qmk_firmware}') + + for path in qmk_userspace_paths(): + try: + qmk_userspace_validate(path) + cli.log.info(f'Testing userspace candidate: {{fg_cyan}}{path}{{fg_reset}} -- {{fg_green}}Valid `qmk.json`') + except FileNotFoundError: + cli.log.warn(f'Testing userspace candidate: {{fg_cyan}}{path}{{fg_reset}} -- {{fg_red}}Missing `qmk.json`') + except UserspaceValidationError as err: + cli.log.warn(f'Testing userspace candidate: {{fg_cyan}}{path}{{fg_reset}} -- {{fg_red}}Invalid `qmk.json`') + cli.log.warn(f' -- {{fg_cyan}}{path}/qmk.json{{fg_reset}} validation error: {err}') + + if QMK_USERSPACE is not None: + cli.log.info(f'QMK userspace: {{fg_cyan}}{QMK_USERSPACE}') + cli.log.info(f'Userspace enabled: {{fg_cyan}}{HAS_QMK_USERSPACE}') + + @cli.argument('-y', '--yes', action='store_true', arg_only=True, help='Answer yes to all questions.') @cli.argument('-n', '--no', action='store_true', arg_only=True, help='Answer no to all questions.') @cli.subcommand('Basic QMK environment checks') @@ -108,6 +128,9 @@ def doctor(cli): cli.log.info('QMK home: {fg_cyan}%s', QMK_FIRMWARE) status = os_status = os_tests() + + userspace_tests(None) + git_status = git_tests() if git_status == CheckStatus.ERROR or (os_status == CheckStatus.OK and git_status == CheckStatus.WARNING): diff --git a/lib/python/qmk/cli/format/json.py b/lib/python/qmk/cli/format/json.py index 3299a0d807..283513254c 100755 --- a/lib/python/qmk/cli/format/json.py +++ b/lib/python/qmk/cli/format/json.py @@ -9,48 +9,74 @@ from milc import cli from qmk.info import info_json from qmk.json_schema import json_load, validate -from qmk.json_encoders import InfoJSONEncoder, KeymapJSONEncoder +from qmk.json_encoders import InfoJSONEncoder, KeymapJSONEncoder, UserspaceJSONEncoder from qmk.path import normpath -@cli.argument('json_file', arg_only=True, type=normpath, help='JSON file to format') -@cli.argument('-f', '--format', choices=['auto', 'keyboard', 'keymap'], default='auto', arg_only=True, help='JSON formatter to use (Default: autodetect)') -@cli.argument('-i', '--inplace', action='store_true', arg_only=True, help='If set, will operate in-place on the input file') -@cli.argument('-p', '--print', action='store_true', arg_only=True, help='If set, will print the formatted json to stdout ') -@cli.subcommand('Generate an info.json file for a keyboard.', hidden=False if cli.config.user.developer else True) -def format_json(cli): - """Format a json file. +def _detect_json_format(file, json_data): + """Detect the format of a json file. """ - json_file = json_load(cli.args.json_file) - - if cli.args.format == 'auto': + json_encoder = None + try: + validate(json_data, 'qmk.user_repo.v1') + json_encoder = UserspaceJSONEncoder + except ValidationError: + pass + + if json_encoder is None: try: - validate(json_file, 'qmk.keyboard.v1') + validate(json_data, 'qmk.keyboard.v1') json_encoder = InfoJSONEncoder - except ValidationError as e: - cli.log.warning('File %s did not validate as a keyboard:\n\t%s', cli.args.json_file, e) - cli.log.info('Treating %s as a keymap file.', cli.args.json_file) + cli.log.warning('File %s did not validate as a keyboard info.json or userspace qmk.json:\n\t%s', file, e) + cli.log.info('Treating %s as a keymap file.', file) json_encoder = KeymapJSONEncoder + + return json_encoder + + +def _get_json_encoder(file, json_data): + """Get the json encoder for a file. + """ + json_encoder = None + if cli.args.format == 'auto': + json_encoder = _detect_json_format(file, json_data) elif cli.args.format == 'keyboard': json_encoder = InfoJSONEncoder elif cli.args.format == 'keymap': json_encoder = KeymapJSONEncoder + elif cli.args.format == 'userspace': + json_encoder = UserspaceJSONEncoder else: # This should be impossible cli.log.error('Unknown format: %s', cli.args.format) + return json_encoder + + +@cli.argument('json_file', arg_only=True, type=normpath, help='JSON file to format') +@cli.argument('-f', '--format', choices=['auto', 'keyboard', 'keymap', 'userspace'], default='auto', arg_only=True, help='JSON formatter to use (Default: autodetect)') +@cli.argument('-i', '--inplace', action='store_true', arg_only=True, help='If set, will operate in-place on the input file') +@cli.argument('-p', '--print', action='store_true', arg_only=True, help='If set, will print the formatted json to stdout ') +@cli.subcommand('Generate an info.json file for a keyboard.', hidden=False if cli.config.user.developer else True) +def format_json(cli): + """Format a json file. + """ + json_data = json_load(cli.args.json_file) + + json_encoder = _get_json_encoder(cli.args.json_file, json_data) + if json_encoder is None: return False - if json_encoder == KeymapJSONEncoder and 'layout' in json_file: + if json_encoder == KeymapJSONEncoder and 'layout' in json_data: # Attempt to format the keycodes. - layout = json_file['layout'] - info_data = info_json(json_file['keyboard']) + layout = json_data['layout'] + info_data = info_json(json_data['keyboard']) if layout in info_data.get('layout_aliases', {}): - layout = json_file['layout'] = info_data['layout_aliases'][layout] + layout = json_data['layout'] = info_data['layout_aliases'][layout] if layout in info_data.get('layouts'): - for layer_num, layer in enumerate(json_file['layers']): + for layer_num, layer in enumerate(json_data['layers']): current_layer = [] last_row = 0 @@ -61,9 +87,9 @@ def format_json(cli): current_layer.append(keymap_key) - json_file['layers'][layer_num] = current_layer + json_data['layers'][layer_num] = current_layer - output = json.dumps(json_file, cls=json_encoder, sort_keys=True) + output = json.dumps(json_data, cls=json_encoder, sort_keys=True) if cli.args.inplace: with open(cli.args.json_file, 'w+', encoding='utf-8') as outfile: diff --git a/lib/python/qmk/cli/mass_compile.py b/lib/python/qmk/cli/mass_compile.py index 7968de53e7..b025f85701 100755 --- a/lib/python/qmk/cli/mass_compile.py +++ b/lib/python/qmk/cli/mass_compile.py @@ -72,7 +72,7 @@ all: {keyboard_safe}_{keymap_name}_binary # yapf: enable f.write('\n') - cli.run([make_cmd, *get_make_parallel_args(parallel), '-f', makefile.as_posix(), 'all'], capture_output=False, stdin=DEVNULL) + cli.run([find_make(), *get_make_parallel_args(parallel), '-f', makefile.as_posix(), 'all'], capture_output=False, stdin=DEVNULL) # Check for failures failures = [f for f in builddir.glob(f'failed.log.{os.getpid()}.*')] diff --git a/lib/python/qmk/cli/new/keymap.py b/lib/python/qmk/cli/new/keymap.py index 9b0ac221a4..d4339bc9ef 100755 --- a/lib/python/qmk/cli/new/keymap.py +++ b/lib/python/qmk/cli/new/keymap.py @@ -5,10 +5,12 @@ import shutil from milc import cli from milc.questions import question +from qmk.constants import HAS_QMK_USERSPACE, QMK_USERSPACE from qmk.path import is_keyboard, keymaps, keymap from qmk.git import git_get_username from qmk.decorators import automagic_keyboard, automagic_keymap from qmk.keyboard import keyboard_completer, keyboard_folder +from qmk.userspace import UserspaceDefs def prompt_keyboard(): @@ -68,3 +70,9 @@ def new_keymap(cli): # end message to user cli.log.info(f'{{fg_green}}Created a new keymap called {{fg_cyan}}{user_name}{{fg_green}} in: {{fg_cyan}}{keymap_path_new}.{{fg_reset}}') cli.log.info(f"Compile a firmware with your new keymap by typing: {{fg_yellow}}qmk compile -kb {kb_name} -km {user_name}{{fg_reset}}.") + + # Add to userspace compile if we have userspace available + if HAS_QMK_USERSPACE: + userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json') + userspace.add_target(keyboard=kb_name, keymap=user_name, do_print=False) + return userspace.save() diff --git a/lib/python/qmk/cli/userspace/__init__.py b/lib/python/qmk/cli/userspace/__init__.py new file mode 100644 index 0000000000..5757d3a4c9 --- /dev/null +++ b/lib/python/qmk/cli/userspace/__init__.py @@ -0,0 +1,5 @@ +from . import doctor +from . import add +from . import remove +from . import list +from . import compile diff --git a/lib/python/qmk/cli/userspace/add.py b/lib/python/qmk/cli/userspace/add.py new file mode 100644 index 0000000000..8993d54dba --- /dev/null +++ b/lib/python/qmk/cli/userspace/add.py @@ -0,0 +1,51 @@ +# Copyright 2023 Nick Brassel (@tzarc) +# SPDX-License-Identifier: GPL-2.0-or-later +from pathlib import Path +from milc import cli + +from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE +from qmk.keyboard import keyboard_completer, keyboard_folder_or_all +from qmk.keymap import keymap_completer, is_keymap_target +from qmk.userspace import UserspaceDefs + + +@cli.argument('builds', nargs='*', arg_only=True, help="List of builds in form <keyboard>:<keymap>, or path to a keymap JSON file.") +@cli.argument('-kb', '--keyboard', type=keyboard_folder_or_all, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') +@cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.') +@cli.subcommand('Adds a build target to userspace `qmk.json`.') +def userspace_add(cli): + if not HAS_QMK_USERSPACE: + cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.') + return False + + userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json') + + if len(cli.args.builds) > 0: + json_like_targets = list([Path(p) for p in filter(lambda e: Path(e).exists() and Path(e).suffix == '.json', cli.args.builds)]) + make_like_targets = list(filter(lambda e: Path(e) not in json_like_targets, cli.args.builds)) + + for e in json_like_targets: + userspace.add_target(json_path=e) + + for e in make_like_targets: + s = e.split(':') + userspace.add_target(keyboard=s[0], keymap=s[1]) + + else: + failed = False + try: + if not is_keymap_target(cli.args.keyboard, cli.args.keymap): + failed = True + except KeyError: + failed = True + + if failed: + from qmk.cli.new.keymap import new_keymap + cli.config.new_keymap.keyboard = cli.args.keyboard + cli.config.new_keymap.keymap = cli.args.keymap + if new_keymap(cli) is not False: + userspace.add_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap) + else: + userspace.add_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap) + + return userspace.save() diff --git a/lib/python/qmk/cli/userspace/compile.py b/lib/python/qmk/cli/userspace/compile.py new file mode 100644 index 0000000000..0a42dd5bf5 --- /dev/null +++ b/lib/python/qmk/cli/userspace/compile.py @@ -0,0 +1,38 @@ +# Copyright 2023 Nick Brassel (@tzarc) +# SPDX-License-Identifier: GPL-2.0-or-later +from pathlib import Path +from milc import cli + +from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE +from qmk.commands import build_environment +from qmk.userspace import UserspaceDefs +from qmk.build_targets import JsonKeymapBuildTarget +from qmk.search import search_keymap_targets +from qmk.cli.mass_compile import mass_compile_targets + + +@cli.argument('-t', '--no-temp', arg_only=True, action='store_true', help="Remove temporary files during build.") +@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.") +@cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.") +@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the commands to be run.") +@cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.") +@cli.subcommand('Compiles the build targets specified in userspace `qmk.json`.') +def userspace_compile(cli): + if not HAS_QMK_USERSPACE: + cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.') + return False + + userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json') + + build_targets = [] + keyboard_keymap_targets = [] + for e in userspace.build_targets: + if isinstance(e, Path): + build_targets.append(JsonKeymapBuildTarget(e)) + elif isinstance(e, dict): + keyboard_keymap_targets.append((e['keyboard'], e['keymap'])) + + if len(keyboard_keymap_targets) > 0: + build_targets.extend(search_keymap_targets(keyboard_keymap_targets)) + + mass_compile_targets(list(set(build_targets)), cli.args.clean, cli.args.dry_run, cli.config.userspace_compile.no_temp, cli.config.userspace_compile.parallel, **build_environment(cli.args.env)) diff --git a/lib/python/qmk/cli/userspace/doctor.py b/lib/python/qmk/cli/userspace/doctor.py new file mode 100644 index 0000000000..2b7e29aa7e --- /dev/null +++ b/lib/python/qmk/cli/userspace/doctor.py @@ -0,0 +1,11 @@ +# Copyright 2023 Nick Brassel (@tzarc) +# SPDX-License-Identifier: GPL-2.0-or-later +from milc import cli + +from qmk.constants import QMK_FIRMWARE +from qmk.cli.doctor.main import userspace_tests + + +@cli.subcommand('Checks userspace configuration.') +def userspace_doctor(cli): + userspace_tests(QMK_FIRMWARE) diff --git a/lib/python/qmk/cli/userspace/list.py b/lib/python/qmk/cli/userspace/list.py new file mode 100644 index 0000000000..a63f669dd7 --- /dev/null +++ b/lib/python/qmk/cli/userspace/list.py @@ -0,0 +1,51 @@ +# Copyright 2023 Nick Brassel (@tzarc) +# SPDX-License-Identifier: GPL-2.0-or-later +from pathlib import Path +from dotty_dict import Dotty +from milc import cli + +from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE +from qmk.userspace import UserspaceDefs +from qmk.build_targets import BuildTarget +from qmk.keyboard import is_all_keyboards, keyboard_folder +from qmk.keymap import is_keymap_target +from qmk.search import search_keymap_targets + + +@cli.argument('-e', '--expand', arg_only=True, action='store_true', help="Expands any use of `all` for either keyboard or keymap.") +@cli.subcommand('Lists the build targets specified in userspace `qmk.json`.') +def userspace_list(cli): + if not HAS_QMK_USERSPACE: + cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.') + return False + + userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json') + + if cli.args.expand: + build_targets = [] + for e in userspace.build_targets: + if isinstance(e, Path): + build_targets.append(e) + elif isinstance(e, dict) or isinstance(e, Dotty): + build_targets.extend(search_keymap_targets([(e['keyboard'], e['keymap'])])) + else: + build_targets = userspace.build_targets + + for e in build_targets: + if isinstance(e, Path): + # JSON keymap from userspace + cli.log.info(f'JSON keymap: {{fg_cyan}}{e}{{fg_reset}}') + continue + elif isinstance(e, dict) or isinstance(e, Dotty): + # keyboard/keymap dict from userspace + keyboard = e['keyboard'] + keymap = e['keymap'] + elif isinstance(e, BuildTarget): + # BuildTarget from search_keymap_targets() + keyboard = e.keyboard + keymap = e.keymap + + if is_all_keyboards(keyboard) or is_keymap_target(keyboard_folder(keyboard), keymap): + cli.log.info(f'Keyboard: {{fg_cyan}}{keyboard}{{fg_reset}}, keymap: {{fg_cyan}}{keymap}{{fg_reset}}') + else: + cli.log.warn(f'Keyboard: {{fg_cyan}}{keyboard}{{fg_reset}}, keymap: {{fg_cyan}}{keymap}{{fg_reset}} -- not found!') diff --git a/lib/python/qmk/cli/userspace/remove.py b/lib/python/qmk/cli/userspace/remove.py new file mode 100644 index 0000000000..c7d180bfd1 --- /dev/null +++ b/lib/python/qmk/cli/userspace/remove.py @@ -0,0 +1,37 @@ +# Copyright 2023 Nick Brassel (@tzarc) +# SPDX-License-Identifier: GPL-2.0-or-later +from pathlib import Path +from milc import cli + +from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE +from qmk.keyboard import keyboard_completer, keyboard_folder_or_all +from qmk.keymap import keymap_completer +from qmk.userspace import UserspaceDefs + + +@cli.argument('builds', nargs='*', arg_only=True, help="List of builds in form <keyboard>:<keymap>, or path to a keymap JSON file.") +@cli.argument('-kb', '--keyboard', type=keyboard_folder_or_all, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') +@cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.') +@cli.subcommand('Removes a build target from userspace `qmk.json`.') +def userspace_remove(cli): + if not HAS_QMK_USERSPACE: + cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.') + return False + + userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json') + + if len(cli.args.builds) > 0: + json_like_targets = list([Path(p) for p in filter(lambda e: Path(e).exists() and Path(e).suffix == '.json', cli.args.builds)]) + make_like_targets = list(filter(lambda e: Path(e) not in json_like_targets, cli.args.builds)) + + for e in json_like_targets: + userspace.remove_target(json_path=e) + + for e in make_like_targets: + s = e.split(':') + userspace.remove_target(keyboard=s[0], keymap=s[1]) + + else: + userspace.remove_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap) + + return userspace.save() diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py index 519cb4c708..d95ff5f923 100644 --- a/lib/python/qmk/commands.py +++ b/lib/python/qmk/commands.py @@ -3,10 +3,12 @@ import os import sys import shutil +from pathlib import Path from milc import cli import jsonschema +from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE from qmk.json_schema import json_load, validate from qmk.keyboard import keyboard_alias_definitions @@ -75,6 +77,10 @@ def build_environment(args): envs[key] = value else: cli.log.warning('Invalid environment variable: %s', env) + + if HAS_QMK_USERSPACE: + envs['QMK_USERSPACE'] = Path(QMK_USERSPACE).resolve() + return envs diff --git a/lib/python/qmk/constants.py b/lib/python/qmk/constants.py index 1967441fc8..90e4452f2b 100644 --- a/lib/python/qmk/constants.py +++ b/lib/python/qmk/constants.py @@ -4,9 +4,17 @@ from os import environ from datetime import date from pathlib import Path +from qmk.userspace import detect_qmk_userspace + # The root of the qmk_firmware tree. QMK_FIRMWARE = Path.cwd() +# The detected userspace tree +QMK_USERSPACE = detect_qmk_userspace() + +# Whether or not we have a separate userspace directory +HAS_QMK_USERSPACE = True if QMK_USERSPACE is not None else False + # Upstream repo url QMK_FIRMWARE_UPSTREAM = 'qmk/qmk_firmware' diff --git a/lib/python/qmk/json_encoders.py b/lib/python/qmk/json_encoders.py index 1e90f6a288..0e4ad1d220 100755 --- a/lib/python/qmk/json_encoders.py +++ b/lib/python/qmk/json_encoders.py @@ -217,3 +217,21 @@ class KeymapJSONEncoder(QMKJSONEncoder): return '50' + str(key) return key + + +class UserspaceJSONEncoder(QMKJSONEncoder): + """Custom encoder to make userspace qmk.json's a little nicer to work with. + """ + def sort_dict(self, item): + """Sorts the hashes in a nice way. + """ + key = item[0] + + if self.indentation_level == 1: + if key == 'userspace_version': + return '00userspace_version' + + if key == 'build_targets': + return '01build_targets' + + return key diff --git a/lib/python/qmk/keyboard.py b/lib/python/qmk/keyboard.py index 34257bee8d..b56505d649 100644 --- a/lib/python/qmk/keyboard.py +++ b/lib/python/qmk/keyboard.py @@ -78,13 +78,17 @@ def keyboard_alias_definitions(): def is_all_keyboards(keyboard): """Returns True if the keyboard is an AllKeyboards object. """ + if isinstance(keyboard, str): + return (keyboard == 'all') return isinstance(keyboard, AllKeyboards) def find_keyboard_from_dir(): """Returns a keyboard name based on the user's current directory. """ - relative_cwd = qmk.path.under_qmk_firmware() + relative_cwd = qmk.path.under_qmk_userspace() + if not relative_cwd: + relative_cwd = qmk.path.under_qmk_firmware() if relative_cwd and len(relative_cwd.parts) > 1 and relative_cwd.parts[0] == 'keyboards': # Attempt to extract the keyboard name from the current directory @@ -133,6 +137,22 @@ def keyboard_folder(keyboard): return keyboard +def keyboard_aliases(keyboard): + """Returns the list of aliases for the supplied keyboard. + + Includes the keyboard itself. + """ + aliases = json_load(Path('data/mappings/keyboard_aliases.hjson')) + + if keyboard in aliases: + keyboard = aliases[keyboard].get('target', keyboard) + + keyboards = set(filter(lambda k: aliases[k].get('target', '') == keyboard, aliases.keys())) + keyboards.add(keyboard) + keyboards = list(sorted(keyboards)) + return keyboards + + def keyboard_folder_or_all(keyboard): """Returns the actual keyboard folder. diff --git a/lib/python/qmk/keymap.py b/lib/python/qmk/keymap.py index 281c53cfda..b7bf897377 100644 --- a/lib/python/qmk/keymap.py +++ b/lib/python/qmk/keymap.py @@ -12,7 +12,8 @@ from pygments.token import Token from pygments import lex import qmk.path -from qmk.keyboard import find_keyboard_from_dir, keyboard_folder +from qmk.constants import QMK_FIRMWARE, QMK_USERSPACE, HAS_QMK_USERSPACE +from qmk.keyboard import find_keyboard_from_dir, keyboard_folder, keyboard_aliases from qmk.errors import CppError from qmk.info import info_json @@ -194,29 +195,38 @@ def _strip_any(keycode): def find_keymap_from_dir(*args): """Returns `(keymap_name, source)` for the directory provided (or cwd if not specified). """ - relative_path = qmk.path.under_qmk_firmware(*args) + def _impl_find_keymap_from_dir(relative_path): + if relative_path and len(relative_path.parts) > 1: + # If we're in `qmk_firmware/keyboards` and `keymaps` is in our path, try to find the keyboard name. + if relative_path.parts[0] == 'keyboards' and 'keymaps' in relative_path.parts: + current_path = Path('/'.join(relative_path.parts[1:])) # Strip 'keyboards' from the front - if relative_path and len(relative_path.parts) > 1: - # If we're in `qmk_firmware/keyboards` and `keymaps` is in our path, try to find the keyboard name. - if relative_path.parts[0] == 'keyboards' and 'keymaps' in relative_path.parts: - current_path = Path('/'.join(relative_path.parts[1:])) # Strip 'keyboards' from the front + if 'keymaps' in current_path.parts and current_path.name != 'keymaps': + while current_path.parent.name != 'keymaps': + current_path = current_path.parent - if 'keymaps' in current_path.parts and current_path.name != 'keymaps': - while current_path.parent.name != 'keymaps': - current_path = current_path.parent + return current_path.name, 'keymap_directory' - return current_path.name, 'keymap_directory' + # If we're in `qmk_firmware/layouts` guess the name from the community keymap they're in + elif relative_path.parts[0] == 'layouts' and is_keymap_dir(relative_path): + return relative_path.name, 'layouts_directory' - # If we're in `qmk_firmware/layouts` guess the name from the community keymap they're in - elif relative_path.parts[0] == 'layouts' and is_keymap_dir(relative_path): - return relative_path.name, 'layouts_directory' + # If we're in `qmk_firmware/users` guess the name from the userspace they're in + elif relative_path.parts[0] == 'users': + # Guess the keymap name based on which userspace they're in + return relative_path.parts[1], 'users_directory' + return None, None - # If we're in `qmk_firmware/users` guess the name from the userspace they're in - elif relative_path.parts[0] == 'users': - # Guess the keymap name based on which userspace they're in - return relative_path.parts[1], 'users_directory' + if HAS_QMK_USERSPACE: + name, source = _impl_find_keymap_from_dir(qmk.path.under_qmk_userspace(*args)) + if name and source: + return name, source - return None, None + name, source = _impl_find_keymap_from_dir(qmk.path.under_qmk_firmware(*args)) + if name and source: + return name, source + + return (None, None) def keymap_completer(prefix, action, parser, parsed_args): @@ -417,29 +427,45 @@ def locate_keymap(keyboard, keymap): raise KeyError('Invalid keyboard: ' + repr(keyboard)) # Check the keyboard folder first, last match wins - checked_dirs = '' keymap_path = '' - for dir in keyboard_folder(keyboard).split('/'): - if checked_dirs: - checked_dirs = '/'.join((checked_dirs, dir)) - else: - checked_dirs = dir + search_dirs = [QMK_FIRMWARE] + keyboard_dirs = [keyboard_folder(keyboard)] + if HAS_QMK_USERSPACE: + # When we've got userspace, check there _last_ as we want them to override anything in the main repo. + search_dirs.append(QMK_USERSPACE) + # We also want to search for any aliases as QMK's folder structure may have changed, with an alias, but the user + # hasn't updated their keymap location yet. + keyboard_dirs.extend(keyboard_aliases(keyboard)) + keyboard_dirs = list(set(keyboard_dirs)) + + for search_dir in search_dirs: + for keyboard_dir in keyboard_dirs: + checked_dirs = '' + for dir in keyboard_dir.split('/'): + if checked_dirs: + checked_dirs = '/'.join((checked_dirs, dir)) + else: + checked_dirs = dir - keymap_dir = Path('keyboards') / checked_dirs / 'keymaps' + keymap_dir = Path(search_dir) / Path('keyboards') / checked_dirs / 'keymaps' - if (keymap_dir / keymap / 'keymap.c').exists(): - keymap_path = keymap_dir / keymap / 'keymap.c' - if (keymap_dir / keymap / 'keymap.json').exists(): - keymap_path = keymap_dir / keymap / 'keymap.json' + if (keymap_dir / keymap / 'keymap.c').exists(): + keymap_path = keymap_dir / keymap / 'keymap.c' + if (keymap_dir / keymap / 'keymap.json').exists(): + keymap_path = keymap_dir / keymap / 'keymap.json' - if keymap_path: - return keymap_path + if keymap_path: + return keymap_path # Check community layouts as a fallback info = info_json(keyboard) - for community_parent in Path('layouts').glob('*/'): + community_parents = list(Path('layouts').glob('*/')) + if HAS_QMK_USERSPACE and (Path(QMK_USERSPACE) / "layouts").exists(): + community_parents.append(Path(QMK_USERSPACE) / "layouts") + + for community_parent in community_parents: for layout in info.get("community_layouts", []): community_layout = community_parent / layout / keymap if community_layout.exists(): @@ -449,6 +475,16 @@ def locate_keymap(keyboard, keymap): return community_layout / 'keymap.c' +def is_keymap_target(keyboard, keymap): + if keymap == 'all': + return True + + if locate_keymap(keyboard, keymap): + return True + + return False + + def list_keymaps(keyboard, c=True, json=True, additional_files=None, fullpath=False): """List the available keymaps for a keyboard. @@ -473,26 +509,30 @@ def list_keymaps(keyboard, c=True, json=True, additional_files=None, fullpath=Fa """ names = set() - keyboards_dir = Path('keyboards') - kb_path = keyboards_dir / keyboard - # walk up the directory tree until keyboards_dir # and collect all directories' name with keymap.c file in it - while kb_path != keyboards_dir: - keymaps_dir = kb_path / "keymaps" - - if keymaps_dir.is_dir(): - for keymap in keymaps_dir.iterdir(): - if is_keymap_dir(keymap, c, json, additional_files): - keymap = keymap if fullpath else keymap.name - names.add(keymap) + for search_dir in [QMK_FIRMWARE, QMK_USERSPACE] if HAS_QMK_USERSPACE else [QMK_FIRMWARE]: + keyboards_dir = search_dir / Path('keyboards') + kb_path = keyboards_dir / keyboard + + while kb_path != keyboards_dir: + keymaps_dir = kb_path / "keymaps" + if keymaps_dir.is_dir(): + for keymap in keymaps_dir.iterdir(): + if is_keymap_dir(keymap, c, json, additional_files): + keymap = keymap if fullpath else keymap.name + names.add(keymap) - kb_path = kb_path.parent + kb_path = kb_path.parent # Check community layouts as a fallback info = info_json(keyboard) - for community_parent in Path('layouts').glob('*/'): + community_parents = list(Path('layouts').glob('*/')) + if HAS_QMK_USERSPACE and (Path(QMK_USERSPACE) / "layouts").exists(): + community_parents.append(Path(QMK_USERSPACE) / "layouts") + + for community_parent in community_parents: for layout in info.get("community_layouts", []): cl_path = community_parent / layout if cl_path.is_dir(): diff --git a/lib/python/qmk/path.py b/lib/python/qmk/path.py index 94582a05e0..74364ee04b 100644 --- a/lib/python/qmk/path.py +++ b/lib/python/qmk/path.py @@ -5,7 +5,7 @@ import os import argparse from pathlib import Path -from qmk.constants import MAX_KEYBOARD_SUBFOLDERS, QMK_FIRMWARE +from qmk.constants import MAX_KEYBOARD_SUBFOLDERS, QMK_FIRMWARE, QMK_USERSPACE, HAS_QMK_USERSPACE from qmk.errors import NoSuchKeyboardError @@ -28,6 +28,40 @@ def under_qmk_firmware(path=Path(os.environ['ORIG_CWD'])): return None +def under_qmk_userspace(path=Path(os.environ['ORIG_CWD'])): + """Returns a Path object representing the relative path under $QMK_USERSPACE, or None. + """ + try: + if HAS_QMK_USERSPACE: + return path.relative_to(QMK_USERSPACE) + except ValueError: + pass + return None + + +def is_under_qmk_firmware(path=Path(os.environ['ORIG_CWD'])): + """Returns a boolean if the input path is a child under qmk_firmware. + """ + if path is None: + return False + try: + return Path(os.path.commonpath([Path(path), QMK_FIRMWARE])) == QMK_FIRMWARE + except ValueError: + return False + + +def is_under_qmk_userspace(path=Path(os.environ['ORIG_CWD'])): + """Returns a boolean if the input path is a child under $QMK_USERSPACE. + """ + if path is None: + return False + try: + if HAS_QMK_USERSPACE: + return Path(os.path.commonpath([Path(path), QMK_USERSPACE])) == QMK_USERSPACE + except ValueError: + return False + + def keyboard(keyboard_name): """Returns the path to a keyboard's directory relative to the qmk root. """ @@ -45,11 +79,28 @@ def keymaps(keyboard_name): keyboard_folder = keyboard(keyboard_name) found_dirs = [] + if HAS_QMK_USERSPACE: + this_keyboard_folder = Path(QMK_USERSPACE) / keyboard_folder + for _ in range(MAX_KEYBOARD_SUBFOLDERS): + if (this_keyboard_folder / 'keymaps').exists(): + found_dirs.append((this_keyboard_folder / 'keymaps').resolve()) + + this_keyboard_folder = this_keyboard_folder.parent + if this_keyboard_folder.resolve() == QMK_USERSPACE.resolve(): + break + + # We don't have any relevant keymap directories in userspace, so we'll use the fully-qualified path instead. + if len(found_dirs) == 0: + found_dirs.append((QMK_USERSPACE / keyboard_folder / 'keymaps').resolve()) + + this_keyboard_folder = QMK_FIRMWARE / keyboard_folder for _ in range(MAX_KEYBOARD_SUBFOLDERS): - if (keyboard_folder / 'keymaps').exists(): - found_dirs.append((keyboard_folder / 'keymaps').resolve()) + if (this_keyboard_folder / 'keymaps').exists(): + found_dirs.append((this_keyboard_folder / 'keymaps').resolve()) - keyboard_folder = keyboard_folder.parent + this_keyboard_folder = this_keyboard_folder.parent + if this_keyboard_folder.resolve() == QMK_FIRMWARE.resolve(): + break if len(found_dirs) > 0: return found_dirs diff --git a/lib/python/qmk/userspace.py b/lib/python/qmk/userspace.py new file mode 100644 index 0000000000..3783568006 --- /dev/null +++ b/lib/python/qmk/userspace.py @@ -0,0 +1,185 @@ +# Copyright 2023 Nick Brassel (@tzarc) +# SPDX-License-Identifier: GPL-2.0-or-later +from os import environ +from pathlib import Path +import json +import jsonschema + +from milc import cli + +from qmk.json_schema import validate, json_load +from qmk.json_encoders import UserspaceJSONEncoder + + +def qmk_userspace_paths(): + test_dirs = [] + + # If we're already in a directory with a qmk.json and a keyboards or layouts directory, interpret it as userspace + current_dir = Path(environ['ORIG_CWD']) + while len(current_dir.parts) > 1: + if (current_dir / 'qmk.json').is_file(): + test_dirs.append(current_dir) + current_dir = current_dir.parent + + # If we have a QMK_USERSPACE environment variable, use that + if environ.get('QMK_USERSPACE') is not None: + current_dir = Path(environ.get('QMK_USERSPACE')) + if current_dir.is_dir(): + test_dirs.append(current_dir) + + # If someone has configured a directory, use that + if cli.config.user.overlay_dir is not None: + current_dir = Path(cli.config.user.overlay_dir) + if current_dir.is_dir(): + test_dirs.append(current_dir) + + return test_dirs + + +def qmk_userspace_validate(path): + # Construct a UserspaceDefs object to ensure it validates correctly + if (path / 'qmk.json').is_file(): + UserspaceDefs(path / 'qmk.json') + return + + # No qmk.json file found + raise FileNotFoundError('No qmk.json file found.') + + +def detect_qmk_userspace(): + # Iterate through all the detected userspace paths and return the first one that validates correctly + test_dirs = qmk_userspace_paths() + for test_dir in test_dirs: + try: + qmk_userspace_validate(test_dir) + return test_dir + except FileNotFoundError: + continue + except UserspaceValidationError: + continue + return None + + +class UserspaceDefs: + def __init__(self, userspace_json: Path): + self.path = userspace_json + self.build_targets = [] + json = json_load(userspace_json) + + exception = UserspaceValidationError() + success = False + + try: + validate(json, 'qmk.user_repo.v0') # `qmk.json` must have a userspace_version at minimum + except jsonschema.ValidationError as err: + exception.add('qmk.user_repo.v0', err) + raise exception + + # Iterate through each version of the schema, starting with the latest and decreasing to v1 + try: + validate(json, 'qmk.user_repo.v1') + self.__load_v1(json) + success = True + except jsonschema.ValidationError as err: + exception.add('qmk.user_repo.v1', err) + + if not success: + raise exception + + def save(self): + target_json = { + "userspace_version": "1.0", # Needs to match latest version + "build_targets": [] + } + + for e in self.build_targets: + if isinstance(e, dict): + target_json['build_targets'].append([e['keyboard'], e['keymap']]) + elif isinstance(e, Path): + target_json['build_targets'].append(str(e.relative_to(self.path.parent))) + + try: + # Ensure what we're writing validates against the latest version of the schema + validate(target_json, 'qmk.user_repo.v1') + except jsonschema.ValidationError as err: + cli.log.error(f'Could not save userspace file: {err}') + return False + + # Only actually write out data if it changed + old_data = json.dumps(json.loads(self.path.read_text()), cls=UserspaceJSONEncoder, sort_keys=True) + new_data = json.dumps(target_json, cls=UserspaceJSONEncoder, sort_keys=True) + if old_data != new_data: + self.path.write_text(new_data) + cli.log.info(f'Saved userspace file to {self.path}.') + return True + + def add_target(self, keyboard=None, keymap=None, json_path=None, do_print=True): + if json_path is not None: + # Assume we're adding a json filename/path + json_path = Path(json_path) + if json_path not in self.build_targets: + self.build_targets.append(json_path) + if do_print: + cli.log.info(f'Added {json_path} to userspace build targets.') + else: + cli.log.info(f'{json_path} is already a userspace build target.') + + elif keyboard is not None and keymap is not None: + # Both keyboard/keymap specified + e = {"keyboard": keyboard, "keymap": keymap} + if e not in self.build_targets: + self.build_targets.append(e) + if do_print: + cli.log.info(f'Added {keyboard}:{keymap} to userspace build targets.') + else: + if do_print: + cli.log.info(f'{keyboard}:{keymap} is already a userspace build target.') + + def remove_target(self, keyboard=None, keymap=None, json_path=None, do_print=True): + if json_path is not None: + # Assume we're removing a json filename/path + json_path = Path(json_path) + if json_path in self.build_targets: + self.build_targets.remove(json_path) + if do_print: + cli.log.info(f'Removed {json_path} from userspace build targets.') + else: + cli.log.info(f'{json_path} is not a userspace build target.') + + elif keyboard is not None and keymap is not None: + # Both keyboard/keymap specified + e = {"keyboard": keyboard, "keymap": keymap} + if e in self.build_targets: + self.build_targets.remove(e) + if do_print: + cli.log.info(f'Removed {keyboard}:{keymap} from userspace build targets.') + else: + if do_print: + cli.log.info(f'{keyboard}:{keymap} is not a userspace build target.') + + def __load_v1(self, json): + for e in json['build_targets']: + if isinstance(e, list) and len(e) == 2: + self.add_target(keyboard=e[0], keymap=e[1], do_print=False) + if isinstance(e, str): + p = self.path.parent / e + if p.exists() and p.suffix == '.json': + self.add_target(json_path=p, do_print=False) + + +class UserspaceValidationError(Exception): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.__exceptions = [] + + def __str__(self): + return self.message + + @property + def exceptions(self): + return self.__exceptions + + def add(self, schema, exception): + self.__exceptions.append((schema, exception)) + errorlist = "\n\n".join([f"{schema}: {exception}" for schema, exception in self.__exceptions]) + self.message = f'Could not validate against any version of the userspace schema. Errors:\n\n{errorlist}' |