diff options
Diffstat (limited to 'lib/python/qmk')
| -rw-r--r-- | lib/python/qmk/build_targets.py | 16 | ||||
| -rw-r--r-- | lib/python/qmk/cli/__init__.py | 5 | ||||
| -rwxr-xr-x | lib/python/qmk/cli/compile.py | 4 | ||||
| -rwxr-xr-x | lib/python/qmk/cli/doctor/main.py | 25 | ||||
| -rwxr-xr-x | lib/python/qmk/cli/format/json.py | 70 | ||||
| -rwxr-xr-x | lib/python/qmk/cli/mass_compile.py | 2 | ||||
| -rwxr-xr-x | lib/python/qmk/cli/new/keymap.py | 8 | ||||
| -rw-r--r-- | lib/python/qmk/cli/userspace/__init__.py | 5 | ||||
| -rw-r--r-- | lib/python/qmk/cli/userspace/add.py | 51 | ||||
| -rw-r--r-- | lib/python/qmk/cli/userspace/compile.py | 38 | ||||
| -rw-r--r-- | lib/python/qmk/cli/userspace/doctor.py | 11 | ||||
| -rw-r--r-- | lib/python/qmk/cli/userspace/list.py | 51 | ||||
| -rw-r--r-- | lib/python/qmk/cli/userspace/remove.py | 37 | ||||
| -rw-r--r-- | lib/python/qmk/commands.py | 6 | ||||
| -rw-r--r-- | lib/python/qmk/constants.py | 8 | ||||
| -rwxr-xr-x | lib/python/qmk/json_encoders.py | 18 | ||||
| -rw-r--r-- | lib/python/qmk/keyboard.py | 22 | ||||
| -rw-r--r-- | lib/python/qmk/keymap.py | 130 | ||||
| -rw-r--r-- | lib/python/qmk/path.py | 59 | ||||
| -rw-r--r-- | lib/python/qmk/userspace.py | 185 | 
20 files changed, 676 insertions, 75 deletions
diff --git a/lib/python/qmk/build_targets.py b/lib/python/qmk/build_targets.py index 16a7ef87a2..1ab489cec3 100644 --- a/lib/python/qmk/build_targets.py +++ b/lib/python/qmk/build_targets.py @@ -10,6 +10,8 @@ from qmk.constants import QMK_FIRMWARE, INTERMEDIATE_OUTPUT_PREFIX  from qmk.commands import find_make, get_make_parallel_args, parse_configurator_json  from qmk.keyboard import keyboard_folder  from qmk.info import keymap_json +from qmk.keymap import locate_keymap +from qmk.path import is_under_qmk_firmware, is_under_qmk_userspace  class BuildTarget: @@ -158,6 +160,20 @@ class KeyboardKeymapBuildTarget(BuildTarget):          for key, value in env_vars.items():              compile_args.append(f'{key}={value}') +        # Need to override the keymap path if the keymap is a userspace directory. +        # This also ensures keyboard aliases as per `keyboard_aliases.hjson` still work if the userspace has the keymap +        # in an equivalent historical location. +        keymap_location = locate_keymap(self.keyboard, self.keymap) +        if is_under_qmk_userspace(keymap_location) and not is_under_qmk_firmware(keymap_location): +            keymap_directory = keymap_location.parent +            compile_args.extend([ +                f'MAIN_KEYMAP_PATH_1={keymap_directory}', +                f'MAIN_KEYMAP_PATH_2={keymap_directory}', +                f'MAIN_KEYMAP_PATH_3={keymap_directory}', +                f'MAIN_KEYMAP_PATH_4={keymap_directory}', +                f'MAIN_KEYMAP_PATH_5={keymap_directory}', +            ]) +          return compile_args diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py index 695a180066..cf60903687 100644 --- a/lib/python/qmk/cli/__init__.py +++ b/lib/python/qmk/cli/__init__.py @@ -81,6 +81,11 @@ subcommands = [      'qmk.cli.new.keymap',      'qmk.cli.painter',      'qmk.cli.pytest', +    'qmk.cli.userspace.add', +    'qmk.cli.userspace.compile', +    'qmk.cli.userspace.doctor', +    'qmk.cli.userspace.list', +    'qmk.cli.userspace.remove',      'qmk.cli.via2json',  ] diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py index 71c1dec162..3c8f3664ea 100755 --- a/lib/python/qmk/cli/compile.py +++ b/lib/python/qmk/cli/compile.py @@ -37,7 +37,9 @@ def compile(cli):          from .mass_compile import mass_compile          cli.args.builds = []          cli.args.filter = [] -        cli.args.no_temp = False +        cli.config.mass_compile.keymap = cli.config.compile.keymap +        cli.config.mass_compile.parallel = cli.config.compile.parallel +        cli.config.mass_compile.no_temp = False          return mass_compile(cli)      # Build the environment vars diff --git a/lib/python/qmk/cli/doctor/main.py b/lib/python/qmk/cli/doctor/main.py index 6a6feb87d1..dd8b58b2c7 100755 --- a/lib/python/qmk/cli/doctor/main.py +++ b/lib/python/qmk/cli/doctor/main.py @@ -9,10 +9,11 @@ from milc import cli  from milc.questions import yesno  from qmk import submodules -from qmk.constants import QMK_FIRMWARE, QMK_FIRMWARE_UPSTREAM +from qmk.constants import QMK_FIRMWARE, QMK_FIRMWARE_UPSTREAM, QMK_USERSPACE, HAS_QMK_USERSPACE  from .check import CheckStatus, check_binaries, check_binary_versions, check_submodules  from qmk.git import git_check_repo, git_get_branch, git_get_tag, git_get_last_log_entry, git_get_common_ancestor, git_is_dirty, git_get_remotes, git_check_deviation  from qmk.commands import in_virtualenv +from qmk.userspace import qmk_userspace_paths, qmk_userspace_validate, UserspaceValidationError  def os_tests(): @@ -92,6 +93,25 @@ def output_submodule_status():                  cli.log.error(f'- {sub_name}: <<< missing or unknown >>>') +def userspace_tests(qmk_firmware): +    if qmk_firmware: +        cli.log.info(f'QMK home: {{fg_cyan}}{qmk_firmware}') + +    for path in qmk_userspace_paths(): +        try: +            qmk_userspace_validate(path) +            cli.log.info(f'Testing userspace candidate: {{fg_cyan}}{path}{{fg_reset}} -- {{fg_green}}Valid `qmk.json`') +        except FileNotFoundError: +            cli.log.warn(f'Testing userspace candidate: {{fg_cyan}}{path}{{fg_reset}} -- {{fg_red}}Missing `qmk.json`') +        except UserspaceValidationError as err: +            cli.log.warn(f'Testing userspace candidate: {{fg_cyan}}{path}{{fg_reset}} -- {{fg_red}}Invalid `qmk.json`') +            cli.log.warn(f' -- {{fg_cyan}}{path}/qmk.json{{fg_reset}} validation error: {err}') + +    if QMK_USERSPACE is not None: +        cli.log.info(f'QMK userspace: {{fg_cyan}}{QMK_USERSPACE}') +    cli.log.info(f'Userspace enabled: {{fg_cyan}}{HAS_QMK_USERSPACE}') + +  @cli.argument('-y', '--yes', action='store_true', arg_only=True, help='Answer yes to all questions.')  @cli.argument('-n', '--no', action='store_true', arg_only=True, help='Answer no to all questions.')  @cli.subcommand('Basic QMK environment checks') @@ -108,6 +128,9 @@ def doctor(cli):      cli.log.info('QMK home: {fg_cyan}%s', QMK_FIRMWARE)      status = os_status = os_tests() + +    userspace_tests(None) +      git_status = git_tests()      if git_status == CheckStatus.ERROR or (os_status == CheckStatus.OK and git_status == CheckStatus.WARNING): diff --git a/lib/python/qmk/cli/format/json.py b/lib/python/qmk/cli/format/json.py index 3299a0d807..283513254c 100755 --- a/lib/python/qmk/cli/format/json.py +++ b/lib/python/qmk/cli/format/json.py @@ -9,48 +9,74 @@ from milc import cli  from qmk.info import info_json  from qmk.json_schema import json_load, validate -from qmk.json_encoders import InfoJSONEncoder, KeymapJSONEncoder +from qmk.json_encoders import InfoJSONEncoder, KeymapJSONEncoder, UserspaceJSONEncoder  from qmk.path import normpath -@cli.argument('json_file', arg_only=True, type=normpath, help='JSON file to format') -@cli.argument('-f', '--format', choices=['auto', 'keyboard', 'keymap'], default='auto', arg_only=True, help='JSON formatter to use (Default: autodetect)') -@cli.argument('-i', '--inplace', action='store_true', arg_only=True, help='If set, will operate in-place on the input file') -@cli.argument('-p', '--print', action='store_true', arg_only=True, help='If set, will print the formatted json to stdout ') -@cli.subcommand('Generate an info.json file for a keyboard.', hidden=False if cli.config.user.developer else True) -def format_json(cli): -    """Format a json file. +def _detect_json_format(file, json_data): +    """Detect the format of a json file.      """ -    json_file = json_load(cli.args.json_file) - -    if cli.args.format == 'auto': +    json_encoder = None +    try: +        validate(json_data, 'qmk.user_repo.v1') +        json_encoder = UserspaceJSONEncoder +    except ValidationError: +        pass + +    if json_encoder is None:          try: -            validate(json_file, 'qmk.keyboard.v1') +            validate(json_data, 'qmk.keyboard.v1')              json_encoder = InfoJSONEncoder -          except ValidationError as e: -            cli.log.warning('File %s did not validate as a keyboard:\n\t%s', cli.args.json_file, e) -            cli.log.info('Treating %s as a keymap file.', cli.args.json_file) +            cli.log.warning('File %s did not validate as a keyboard info.json or userspace qmk.json:\n\t%s', file, e) +            cli.log.info('Treating %s as a keymap file.', file)              json_encoder = KeymapJSONEncoder + +    return json_encoder + + +def _get_json_encoder(file, json_data): +    """Get the json encoder for a file. +    """ +    json_encoder = None +    if cli.args.format == 'auto': +        json_encoder = _detect_json_format(file, json_data)      elif cli.args.format == 'keyboard':          json_encoder = InfoJSONEncoder      elif cli.args.format == 'keymap':          json_encoder = KeymapJSONEncoder +    elif cli.args.format == 'userspace': +        json_encoder = UserspaceJSONEncoder      else:          # This should be impossible          cli.log.error('Unknown format: %s', cli.args.format) +    return json_encoder + + +@cli.argument('json_file', arg_only=True, type=normpath, help='JSON file to format') +@cli.argument('-f', '--format', choices=['auto', 'keyboard', 'keymap', 'userspace'], default='auto', arg_only=True, help='JSON formatter to use (Default: autodetect)') +@cli.argument('-i', '--inplace', action='store_true', arg_only=True, help='If set, will operate in-place on the input file') +@cli.argument('-p', '--print', action='store_true', arg_only=True, help='If set, will print the formatted json to stdout ') +@cli.subcommand('Generate an info.json file for a keyboard.', hidden=False if cli.config.user.developer else True) +def format_json(cli): +    """Format a json file. +    """ +    json_data = json_load(cli.args.json_file) + +    json_encoder = _get_json_encoder(cli.args.json_file, json_data) +    if json_encoder is None:          return False -    if json_encoder == KeymapJSONEncoder and 'layout' in json_file: +    if json_encoder == KeymapJSONEncoder and 'layout' in json_data:          # Attempt to format the keycodes. -        layout = json_file['layout'] -        info_data = info_json(json_file['keyboard']) +        layout = json_data['layout'] +        info_data = info_json(json_data['keyboard'])          if layout in info_data.get('layout_aliases', {}): -            layout = json_file['layout'] = info_data['layout_aliases'][layout] +            layout = json_data['layout'] = info_data['layout_aliases'][layout]          if layout in info_data.get('layouts'): -            for layer_num, layer in enumerate(json_file['layers']): +            for layer_num, layer in enumerate(json_data['layers']):                  current_layer = []                  last_row = 0 @@ -61,9 +87,9 @@ def format_json(cli):                      current_layer.append(keymap_key) -                json_file['layers'][layer_num] = current_layer +                json_data['layers'][layer_num] = current_layer -    output = json.dumps(json_file, cls=json_encoder, sort_keys=True) +    output = json.dumps(json_data, cls=json_encoder, sort_keys=True)      if cli.args.inplace:          with open(cli.args.json_file, 'w+', encoding='utf-8') as outfile: diff --git a/lib/python/qmk/cli/mass_compile.py b/lib/python/qmk/cli/mass_compile.py index 7968de53e7..b025f85701 100755 --- a/lib/python/qmk/cli/mass_compile.py +++ b/lib/python/qmk/cli/mass_compile.py @@ -72,7 +72,7 @@ all: {keyboard_safe}_{keymap_name}_binary                      # yapf: enable                  f.write('\n') -        cli.run([make_cmd, *get_make_parallel_args(parallel), '-f', makefile.as_posix(), 'all'], capture_output=False, stdin=DEVNULL) +        cli.run([find_make(), *get_make_parallel_args(parallel), '-f', makefile.as_posix(), 'all'], capture_output=False, stdin=DEVNULL)          # Check for failures          failures = [f for f in builddir.glob(f'failed.log.{os.getpid()}.*')] diff --git a/lib/python/qmk/cli/new/keymap.py b/lib/python/qmk/cli/new/keymap.py index 9b0ac221a4..d4339bc9ef 100755 --- a/lib/python/qmk/cli/new/keymap.py +++ b/lib/python/qmk/cli/new/keymap.py @@ -5,10 +5,12 @@ import shutil  from milc import cli  from milc.questions import question +from qmk.constants import HAS_QMK_USERSPACE, QMK_USERSPACE  from qmk.path import is_keyboard, keymaps, keymap  from qmk.git import git_get_username  from qmk.decorators import automagic_keyboard, automagic_keymap  from qmk.keyboard import keyboard_completer, keyboard_folder +from qmk.userspace import UserspaceDefs  def prompt_keyboard(): @@ -68,3 +70,9 @@ def new_keymap(cli):      # end message to user      cli.log.info(f'{{fg_green}}Created a new keymap called {{fg_cyan}}{user_name}{{fg_green}} in: {{fg_cyan}}{keymap_path_new}.{{fg_reset}}')      cli.log.info(f"Compile a firmware with your new keymap by typing: {{fg_yellow}}qmk compile -kb {kb_name} -km {user_name}{{fg_reset}}.") + +    # Add to userspace compile if we have userspace available +    if HAS_QMK_USERSPACE: +        userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json') +        userspace.add_target(keyboard=kb_name, keymap=user_name, do_print=False) +        return userspace.save() diff --git a/lib/python/qmk/cli/userspace/__init__.py b/lib/python/qmk/cli/userspace/__init__.py new file mode 100644 index 0000000000..5757d3a4c9 --- /dev/null +++ b/lib/python/qmk/cli/userspace/__init__.py @@ -0,0 +1,5 @@ +from . import doctor +from . import add +from . import remove +from . import list +from . import compile diff --git a/lib/python/qmk/cli/userspace/add.py b/lib/python/qmk/cli/userspace/add.py new file mode 100644 index 0000000000..8993d54dba --- /dev/null +++ b/lib/python/qmk/cli/userspace/add.py @@ -0,0 +1,51 @@ +# Copyright 2023 Nick Brassel (@tzarc) +# SPDX-License-Identifier: GPL-2.0-or-later +from pathlib import Path +from milc import cli + +from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE +from qmk.keyboard import keyboard_completer, keyboard_folder_or_all +from qmk.keymap import keymap_completer, is_keymap_target +from qmk.userspace import UserspaceDefs + + +@cli.argument('builds', nargs='*', arg_only=True, help="List of builds in form <keyboard>:<keymap>, or path to a keymap JSON file.") +@cli.argument('-kb', '--keyboard', type=keyboard_folder_or_all, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') +@cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.') +@cli.subcommand('Adds a build target to userspace `qmk.json`.') +def userspace_add(cli): +    if not HAS_QMK_USERSPACE: +        cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.') +        return False + +    userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json') + +    if len(cli.args.builds) > 0: +        json_like_targets = list([Path(p) for p in filter(lambda e: Path(e).exists() and Path(e).suffix == '.json', cli.args.builds)]) +        make_like_targets = list(filter(lambda e: Path(e) not in json_like_targets, cli.args.builds)) + +        for e in json_like_targets: +            userspace.add_target(json_path=e) + +        for e in make_like_targets: +            s = e.split(':') +            userspace.add_target(keyboard=s[0], keymap=s[1]) + +    else: +        failed = False +        try: +            if not is_keymap_target(cli.args.keyboard, cli.args.keymap): +                failed = True +        except KeyError: +            failed = True + +        if failed: +            from qmk.cli.new.keymap import new_keymap +            cli.config.new_keymap.keyboard = cli.args.keyboard +            cli.config.new_keymap.keymap = cli.args.keymap +            if new_keymap(cli) is not False: +                userspace.add_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap) +        else: +            userspace.add_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap) + +    return userspace.save() diff --git a/lib/python/qmk/cli/userspace/compile.py b/lib/python/qmk/cli/userspace/compile.py new file mode 100644 index 0000000000..0a42dd5bf5 --- /dev/null +++ b/lib/python/qmk/cli/userspace/compile.py @@ -0,0 +1,38 @@ +# Copyright 2023 Nick Brassel (@tzarc) +# SPDX-License-Identifier: GPL-2.0-or-later +from pathlib import Path +from milc import cli + +from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE +from qmk.commands import build_environment +from qmk.userspace import UserspaceDefs +from qmk.build_targets import JsonKeymapBuildTarget +from qmk.search import search_keymap_targets +from qmk.cli.mass_compile import mass_compile_targets + + +@cli.argument('-t', '--no-temp', arg_only=True, action='store_true', help="Remove temporary files during build.") +@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.") +@cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.") +@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the commands to be run.") +@cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.") +@cli.subcommand('Compiles the build targets specified in userspace `qmk.json`.') +def userspace_compile(cli): +    if not HAS_QMK_USERSPACE: +        cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.') +        return False + +    userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json') + +    build_targets = [] +    keyboard_keymap_targets = [] +    for e in userspace.build_targets: +        if isinstance(e, Path): +            build_targets.append(JsonKeymapBuildTarget(e)) +        elif isinstance(e, dict): +            keyboard_keymap_targets.append((e['keyboard'], e['keymap'])) + +    if len(keyboard_keymap_targets) > 0: +        build_targets.extend(search_keymap_targets(keyboard_keymap_targets)) + +    mass_compile_targets(list(set(build_targets)), cli.args.clean, cli.args.dry_run, cli.config.userspace_compile.no_temp, cli.config.userspace_compile.parallel, **build_environment(cli.args.env)) diff --git a/lib/python/qmk/cli/userspace/doctor.py b/lib/python/qmk/cli/userspace/doctor.py new file mode 100644 index 0000000000..2b7e29aa7e --- /dev/null +++ b/lib/python/qmk/cli/userspace/doctor.py @@ -0,0 +1,11 @@ +# Copyright 2023 Nick Brassel (@tzarc) +# SPDX-License-Identifier: GPL-2.0-or-later +from milc import cli + +from qmk.constants import QMK_FIRMWARE +from qmk.cli.doctor.main import userspace_tests + + +@cli.subcommand('Checks userspace configuration.') +def userspace_doctor(cli): +    userspace_tests(QMK_FIRMWARE) diff --git a/lib/python/qmk/cli/userspace/list.py b/lib/python/qmk/cli/userspace/list.py new file mode 100644 index 0000000000..a63f669dd7 --- /dev/null +++ b/lib/python/qmk/cli/userspace/list.py @@ -0,0 +1,51 @@ +# Copyright 2023 Nick Brassel (@tzarc) +# SPDX-License-Identifier: GPL-2.0-or-later +from pathlib import Path +from dotty_dict import Dotty +from milc import cli + +from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE +from qmk.userspace import UserspaceDefs +from qmk.build_targets import BuildTarget +from qmk.keyboard import is_all_keyboards, keyboard_folder +from qmk.keymap import is_keymap_target +from qmk.search import search_keymap_targets + + +@cli.argument('-e', '--expand', arg_only=True, action='store_true', help="Expands any use of `all` for either keyboard or keymap.") +@cli.subcommand('Lists the build targets specified in userspace `qmk.json`.') +def userspace_list(cli): +    if not HAS_QMK_USERSPACE: +        cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.') +        return False + +    userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json') + +    if cli.args.expand: +        build_targets = [] +        for e in userspace.build_targets: +            if isinstance(e, Path): +                build_targets.append(e) +            elif isinstance(e, dict) or isinstance(e, Dotty): +                build_targets.extend(search_keymap_targets([(e['keyboard'], e['keymap'])])) +    else: +        build_targets = userspace.build_targets + +    for e in build_targets: +        if isinstance(e, Path): +            # JSON keymap from userspace +            cli.log.info(f'JSON keymap: {{fg_cyan}}{e}{{fg_reset}}') +            continue +        elif isinstance(e, dict) or isinstance(e, Dotty): +            # keyboard/keymap dict from userspace +            keyboard = e['keyboard'] +            keymap = e['keymap'] +        elif isinstance(e, BuildTarget): +            # BuildTarget from search_keymap_targets() +            keyboard = e.keyboard +            keymap = e.keymap + +        if is_all_keyboards(keyboard) or is_keymap_target(keyboard_folder(keyboard), keymap): +            cli.log.info(f'Keyboard: {{fg_cyan}}{keyboard}{{fg_reset}}, keymap: {{fg_cyan}}{keymap}{{fg_reset}}') +        else: +            cli.log.warn(f'Keyboard: {{fg_cyan}}{keyboard}{{fg_reset}}, keymap: {{fg_cyan}}{keymap}{{fg_reset}} -- not found!') diff --git a/lib/python/qmk/cli/userspace/remove.py b/lib/python/qmk/cli/userspace/remove.py new file mode 100644 index 0000000000..c7d180bfd1 --- /dev/null +++ b/lib/python/qmk/cli/userspace/remove.py @@ -0,0 +1,37 @@ +# Copyright 2023 Nick Brassel (@tzarc) +# SPDX-License-Identifier: GPL-2.0-or-later +from pathlib import Path +from milc import cli + +from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE +from qmk.keyboard import keyboard_completer, keyboard_folder_or_all +from qmk.keymap import keymap_completer +from qmk.userspace import UserspaceDefs + + +@cli.argument('builds', nargs='*', arg_only=True, help="List of builds in form <keyboard>:<keymap>, or path to a keymap JSON file.") +@cli.argument('-kb', '--keyboard', type=keyboard_folder_or_all, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') +@cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.') +@cli.subcommand('Removes a build target from userspace `qmk.json`.') +def userspace_remove(cli): +    if not HAS_QMK_USERSPACE: +        cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.') +        return False + +    userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json') + +    if len(cli.args.builds) > 0: +        json_like_targets = list([Path(p) for p in filter(lambda e: Path(e).exists() and Path(e).suffix == '.json', cli.args.builds)]) +        make_like_targets = list(filter(lambda e: Path(e) not in json_like_targets, cli.args.builds)) + +        for e in json_like_targets: +            userspace.remove_target(json_path=e) + +        for e in make_like_targets: +            s = e.split(':') +            userspace.remove_target(keyboard=s[0], keymap=s[1]) + +    else: +        userspace.remove_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap) + +    return userspace.save() diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py index 519cb4c708..d95ff5f923 100644 --- a/lib/python/qmk/commands.py +++ b/lib/python/qmk/commands.py @@ -3,10 +3,12 @@  import os  import sys  import shutil +from pathlib import Path  from milc import cli  import jsonschema +from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE  from qmk.json_schema import json_load, validate  from qmk.keyboard import keyboard_alias_definitions @@ -75,6 +77,10 @@ def build_environment(args):              envs[key] = value          else:              cli.log.warning('Invalid environment variable: %s', env) + +    if HAS_QMK_USERSPACE: +        envs['QMK_USERSPACE'] = Path(QMK_USERSPACE).resolve() +      return envs diff --git a/lib/python/qmk/constants.py b/lib/python/qmk/constants.py index 1967441fc8..90e4452f2b 100644 --- a/lib/python/qmk/constants.py +++ b/lib/python/qmk/constants.py @@ -4,9 +4,17 @@ from os import environ  from datetime import date  from pathlib import Path +from qmk.userspace import detect_qmk_userspace +  # The root of the qmk_firmware tree.  QMK_FIRMWARE = Path.cwd() +# The detected userspace tree +QMK_USERSPACE = detect_qmk_userspace() + +# Whether or not we have a separate userspace directory +HAS_QMK_USERSPACE = True if QMK_USERSPACE is not None else False +  # Upstream repo url  QMK_FIRMWARE_UPSTREAM = 'qmk/qmk_firmware' diff --git a/lib/python/qmk/json_encoders.py b/lib/python/qmk/json_encoders.py index 1e90f6a288..0e4ad1d220 100755 --- a/lib/python/qmk/json_encoders.py +++ b/lib/python/qmk/json_encoders.py @@ -217,3 +217,21 @@ class KeymapJSONEncoder(QMKJSONEncoder):                  return '50' + str(key)          return key + + +class UserspaceJSONEncoder(QMKJSONEncoder): +    """Custom encoder to make userspace qmk.json's a little nicer to work with. +    """ +    def sort_dict(self, item): +        """Sorts the hashes in a nice way. +        """ +        key = item[0] + +        if self.indentation_level == 1: +            if key == 'userspace_version': +                return '00userspace_version' + +            if key == 'build_targets': +                return '01build_targets' + +        return key diff --git a/lib/python/qmk/keyboard.py b/lib/python/qmk/keyboard.py index 34257bee8d..b56505d649 100644 --- a/lib/python/qmk/keyboard.py +++ b/lib/python/qmk/keyboard.py @@ -78,13 +78,17 @@ def keyboard_alias_definitions():  def is_all_keyboards(keyboard):      """Returns True if the keyboard is an AllKeyboards object.      """ +    if isinstance(keyboard, str): +        return (keyboard == 'all')      return isinstance(keyboard, AllKeyboards)  def find_keyboard_from_dir():      """Returns a keyboard name based on the user's current directory.      """ -    relative_cwd = qmk.path.under_qmk_firmware() +    relative_cwd = qmk.path.under_qmk_userspace() +    if not relative_cwd: +        relative_cwd = qmk.path.under_qmk_firmware()      if relative_cwd and len(relative_cwd.parts) > 1 and relative_cwd.parts[0] == 'keyboards':          # Attempt to extract the keyboard name from the current directory @@ -133,6 +137,22 @@ def keyboard_folder(keyboard):      return keyboard +def keyboard_aliases(keyboard): +    """Returns the list of aliases for the supplied keyboard. + +    Includes the keyboard itself. +    """ +    aliases = json_load(Path('data/mappings/keyboard_aliases.hjson')) + +    if keyboard in aliases: +        keyboard = aliases[keyboard].get('target', keyboard) + +    keyboards = set(filter(lambda k: aliases[k].get('target', '') == keyboard, aliases.keys())) +    keyboards.add(keyboard) +    keyboards = list(sorted(keyboards)) +    return keyboards + +  def keyboard_folder_or_all(keyboard):      """Returns the actual keyboard folder. diff --git a/lib/python/qmk/keymap.py b/lib/python/qmk/keymap.py index 281c53cfda..b7bf897377 100644 --- a/lib/python/qmk/keymap.py +++ b/lib/python/qmk/keymap.py @@ -12,7 +12,8 @@ from pygments.token import Token  from pygments import lex  import qmk.path -from qmk.keyboard import find_keyboard_from_dir, keyboard_folder +from qmk.constants import QMK_FIRMWARE, QMK_USERSPACE, HAS_QMK_USERSPACE +from qmk.keyboard import find_keyboard_from_dir, keyboard_folder, keyboard_aliases  from qmk.errors import CppError  from qmk.info import info_json @@ -194,29 +195,38 @@ def _strip_any(keycode):  def find_keymap_from_dir(*args):      """Returns `(keymap_name, source)` for the directory provided (or cwd if not specified).      """ -    relative_path = qmk.path.under_qmk_firmware(*args) +    def _impl_find_keymap_from_dir(relative_path): +        if relative_path and len(relative_path.parts) > 1: +            # If we're in `qmk_firmware/keyboards` and `keymaps` is in our path, try to find the keyboard name. +            if relative_path.parts[0] == 'keyboards' and 'keymaps' in relative_path.parts: +                current_path = Path('/'.join(relative_path.parts[1:]))  # Strip 'keyboards' from the front -    if relative_path and len(relative_path.parts) > 1: -        # If we're in `qmk_firmware/keyboards` and `keymaps` is in our path, try to find the keyboard name. -        if relative_path.parts[0] == 'keyboards' and 'keymaps' in relative_path.parts: -            current_path = Path('/'.join(relative_path.parts[1:]))  # Strip 'keyboards' from the front +                if 'keymaps' in current_path.parts and current_path.name != 'keymaps': +                    while current_path.parent.name != 'keymaps': +                        current_path = current_path.parent -            if 'keymaps' in current_path.parts and current_path.name != 'keymaps': -                while current_path.parent.name != 'keymaps': -                    current_path = current_path.parent +                    return current_path.name, 'keymap_directory' -                return current_path.name, 'keymap_directory' +            # If we're in `qmk_firmware/layouts` guess the name from the community keymap they're in +            elif relative_path.parts[0] == 'layouts' and is_keymap_dir(relative_path): +                return relative_path.name, 'layouts_directory' -        # If we're in `qmk_firmware/layouts` guess the name from the community keymap they're in -        elif relative_path.parts[0] == 'layouts' and is_keymap_dir(relative_path): -            return relative_path.name, 'layouts_directory' +            # If we're in `qmk_firmware/users` guess the name from the userspace they're in +            elif relative_path.parts[0] == 'users': +                # Guess the keymap name based on which userspace they're in +                return relative_path.parts[1], 'users_directory' +        return None, None -        # If we're in `qmk_firmware/users` guess the name from the userspace they're in -        elif relative_path.parts[0] == 'users': -            # Guess the keymap name based on which userspace they're in -            return relative_path.parts[1], 'users_directory' +    if HAS_QMK_USERSPACE: +        name, source = _impl_find_keymap_from_dir(qmk.path.under_qmk_userspace(*args)) +        if name and source: +            return name, source -    return None, None +    name, source = _impl_find_keymap_from_dir(qmk.path.under_qmk_firmware(*args)) +    if name and source: +        return name, source + +    return (None, None)  def keymap_completer(prefix, action, parser, parsed_args): @@ -417,29 +427,45 @@ def locate_keymap(keyboard, keymap):          raise KeyError('Invalid keyboard: ' + repr(keyboard))      # Check the keyboard folder first, last match wins -    checked_dirs = ''      keymap_path = '' -    for dir in keyboard_folder(keyboard).split('/'): -        if checked_dirs: -            checked_dirs = '/'.join((checked_dirs, dir)) -        else: -            checked_dirs = dir +    search_dirs = [QMK_FIRMWARE] +    keyboard_dirs = [keyboard_folder(keyboard)] +    if HAS_QMK_USERSPACE: +        # When we've got userspace, check there _last_ as we want them to override anything in the main repo. +        search_dirs.append(QMK_USERSPACE) +        # We also want to search for any aliases as QMK's folder structure may have changed, with an alias, but the user +        # hasn't updated their keymap location yet. +        keyboard_dirs.extend(keyboard_aliases(keyboard)) +        keyboard_dirs = list(set(keyboard_dirs)) + +    for search_dir in search_dirs: +        for keyboard_dir in keyboard_dirs: +            checked_dirs = '' +            for dir in keyboard_dir.split('/'): +                if checked_dirs: +                    checked_dirs = '/'.join((checked_dirs, dir)) +                else: +                    checked_dirs = dir -        keymap_dir = Path('keyboards') / checked_dirs / 'keymaps' +                keymap_dir = Path(search_dir) / Path('keyboards') / checked_dirs / 'keymaps' -        if (keymap_dir / keymap / 'keymap.c').exists(): -            keymap_path = keymap_dir / keymap / 'keymap.c' -        if (keymap_dir / keymap / 'keymap.json').exists(): -            keymap_path = keymap_dir / keymap / 'keymap.json' +                if (keymap_dir / keymap / 'keymap.c').exists(): +                    keymap_path = keymap_dir / keymap / 'keymap.c' +                if (keymap_dir / keymap / 'keymap.json').exists(): +                    keymap_path = keymap_dir / keymap / 'keymap.json' -    if keymap_path: -        return keymap_path +        if keymap_path: +            return keymap_path      # Check community layouts as a fallback      info = info_json(keyboard) -    for community_parent in Path('layouts').glob('*/'): +    community_parents = list(Path('layouts').glob('*/')) +    if HAS_QMK_USERSPACE and (Path(QMK_USERSPACE) / "layouts").exists(): +        community_parents.append(Path(QMK_USERSPACE) / "layouts") + +    for community_parent in community_parents:          for layout in info.get("community_layouts", []):              community_layout = community_parent / layout / keymap              if community_layout.exists(): @@ -449,6 +475,16 @@ def locate_keymap(keyboard, keymap):                      return community_layout / 'keymap.c' +def is_keymap_target(keyboard, keymap): +    if keymap == 'all': +        return True + +    if locate_keymap(keyboard, keymap): +        return True + +    return False + +  def list_keymaps(keyboard, c=True, json=True, additional_files=None, fullpath=False):      """List the available keymaps for a keyboard. @@ -473,26 +509,30 @@ def list_keymaps(keyboard, c=True, json=True, additional_files=None, fullpath=Fa      """      names = set() -    keyboards_dir = Path('keyboards') -    kb_path = keyboards_dir / keyboard -      # walk up the directory tree until keyboards_dir      # and collect all directories' name with keymap.c file in it -    while kb_path != keyboards_dir: -        keymaps_dir = kb_path / "keymaps" - -        if keymaps_dir.is_dir(): -            for keymap in keymaps_dir.iterdir(): -                if is_keymap_dir(keymap, c, json, additional_files): -                    keymap = keymap if fullpath else keymap.name -                    names.add(keymap) +    for search_dir in [QMK_FIRMWARE, QMK_USERSPACE] if HAS_QMK_USERSPACE else [QMK_FIRMWARE]: +        keyboards_dir = search_dir / Path('keyboards') +        kb_path = keyboards_dir / keyboard + +        while kb_path != keyboards_dir: +            keymaps_dir = kb_path / "keymaps" +            if keymaps_dir.is_dir(): +                for keymap in keymaps_dir.iterdir(): +                    if is_keymap_dir(keymap, c, json, additional_files): +                        keymap = keymap if fullpath else keymap.name +                        names.add(keymap) -        kb_path = kb_path.parent +            kb_path = kb_path.parent      # Check community layouts as a fallback      info = info_json(keyboard) -    for community_parent in Path('layouts').glob('*/'): +    community_parents = list(Path('layouts').glob('*/')) +    if HAS_QMK_USERSPACE and (Path(QMK_USERSPACE) / "layouts").exists(): +        community_parents.append(Path(QMK_USERSPACE) / "layouts") + +    for community_parent in community_parents:          for layout in info.get("community_layouts", []):              cl_path = community_parent / layout              if cl_path.is_dir(): diff --git a/lib/python/qmk/path.py b/lib/python/qmk/path.py index 94582a05e0..74364ee04b 100644 --- a/lib/python/qmk/path.py +++ b/lib/python/qmk/path.py @@ -5,7 +5,7 @@ import os  import argparse  from pathlib import Path -from qmk.constants import MAX_KEYBOARD_SUBFOLDERS, QMK_FIRMWARE +from qmk.constants import MAX_KEYBOARD_SUBFOLDERS, QMK_FIRMWARE, QMK_USERSPACE, HAS_QMK_USERSPACE  from qmk.errors import NoSuchKeyboardError @@ -28,6 +28,40 @@ def under_qmk_firmware(path=Path(os.environ['ORIG_CWD'])):          return None +def under_qmk_userspace(path=Path(os.environ['ORIG_CWD'])): +    """Returns a Path object representing the relative path under $QMK_USERSPACE, or None. +    """ +    try: +        if HAS_QMK_USERSPACE: +            return path.relative_to(QMK_USERSPACE) +    except ValueError: +        pass +    return None + + +def is_under_qmk_firmware(path=Path(os.environ['ORIG_CWD'])): +    """Returns a boolean if the input path is a child under qmk_firmware. +    """ +    if path is None: +        return False +    try: +        return Path(os.path.commonpath([Path(path), QMK_FIRMWARE])) == QMK_FIRMWARE +    except ValueError: +        return False + + +def is_under_qmk_userspace(path=Path(os.environ['ORIG_CWD'])): +    """Returns a boolean if the input path is a child under $QMK_USERSPACE. +    """ +    if path is None: +        return False +    try: +        if HAS_QMK_USERSPACE: +            return Path(os.path.commonpath([Path(path), QMK_USERSPACE])) == QMK_USERSPACE +    except ValueError: +        return False + +  def keyboard(keyboard_name):      """Returns the path to a keyboard's directory relative to the qmk root.      """ @@ -45,11 +79,28 @@ def keymaps(keyboard_name):      keyboard_folder = keyboard(keyboard_name)      found_dirs = [] +    if HAS_QMK_USERSPACE: +        this_keyboard_folder = Path(QMK_USERSPACE) / keyboard_folder +        for _ in range(MAX_KEYBOARD_SUBFOLDERS): +            if (this_keyboard_folder / 'keymaps').exists(): +                found_dirs.append((this_keyboard_folder / 'keymaps').resolve()) + +            this_keyboard_folder = this_keyboard_folder.parent +            if this_keyboard_folder.resolve() == QMK_USERSPACE.resolve(): +                break + +        # We don't have any relevant keymap directories in userspace, so we'll use the fully-qualified path instead. +        if len(found_dirs) == 0: +            found_dirs.append((QMK_USERSPACE / keyboard_folder / 'keymaps').resolve()) + +    this_keyboard_folder = QMK_FIRMWARE / keyboard_folder      for _ in range(MAX_KEYBOARD_SUBFOLDERS): -        if (keyboard_folder / 'keymaps').exists(): -            found_dirs.append((keyboard_folder / 'keymaps').resolve()) +        if (this_keyboard_folder / 'keymaps').exists(): +            found_dirs.append((this_keyboard_folder / 'keymaps').resolve()) -        keyboard_folder = keyboard_folder.parent +        this_keyboard_folder = this_keyboard_folder.parent +        if this_keyboard_folder.resolve() == QMK_FIRMWARE.resolve(): +            break      if len(found_dirs) > 0:          return found_dirs diff --git a/lib/python/qmk/userspace.py b/lib/python/qmk/userspace.py new file mode 100644 index 0000000000..3783568006 --- /dev/null +++ b/lib/python/qmk/userspace.py @@ -0,0 +1,185 @@ +# Copyright 2023 Nick Brassel (@tzarc) +# SPDX-License-Identifier: GPL-2.0-or-later +from os import environ +from pathlib import Path +import json +import jsonschema + +from milc import cli + +from qmk.json_schema import validate, json_load +from qmk.json_encoders import UserspaceJSONEncoder + + +def qmk_userspace_paths(): +    test_dirs = [] + +    # If we're already in a directory with a qmk.json and a keyboards or layouts directory, interpret it as userspace +    current_dir = Path(environ['ORIG_CWD']) +    while len(current_dir.parts) > 1: +        if (current_dir / 'qmk.json').is_file(): +            test_dirs.append(current_dir) +        current_dir = current_dir.parent + +    # If we have a QMK_USERSPACE environment variable, use that +    if environ.get('QMK_USERSPACE') is not None: +        current_dir = Path(environ.get('QMK_USERSPACE')) +        if current_dir.is_dir(): +            test_dirs.append(current_dir) + +    # If someone has configured a directory, use that +    if cli.config.user.overlay_dir is not None: +        current_dir = Path(cli.config.user.overlay_dir) +        if current_dir.is_dir(): +            test_dirs.append(current_dir) + +    return test_dirs + + +def qmk_userspace_validate(path): +    # Construct a UserspaceDefs object to ensure it validates correctly +    if (path / 'qmk.json').is_file(): +        UserspaceDefs(path / 'qmk.json') +        return + +    # No qmk.json file found +    raise FileNotFoundError('No qmk.json file found.') + + +def detect_qmk_userspace(): +    # Iterate through all the detected userspace paths and return the first one that validates correctly +    test_dirs = qmk_userspace_paths() +    for test_dir in test_dirs: +        try: +            qmk_userspace_validate(test_dir) +            return test_dir +        except FileNotFoundError: +            continue +        except UserspaceValidationError: +            continue +    return None + + +class UserspaceDefs: +    def __init__(self, userspace_json: Path): +        self.path = userspace_json +        self.build_targets = [] +        json = json_load(userspace_json) + +        exception = UserspaceValidationError() +        success = False + +        try: +            validate(json, 'qmk.user_repo.v0')  # `qmk.json` must have a userspace_version at minimum +        except jsonschema.ValidationError as err: +            exception.add('qmk.user_repo.v0', err) +            raise exception + +        # Iterate through each version of the schema, starting with the latest and decreasing to v1 +        try: +            validate(json, 'qmk.user_repo.v1') +            self.__load_v1(json) +            success = True +        except jsonschema.ValidationError as err: +            exception.add('qmk.user_repo.v1', err) + +        if not success: +            raise exception + +    def save(self): +        target_json = { +            "userspace_version": "1.0",  # Needs to match latest version +            "build_targets": [] +        } + +        for e in self.build_targets: +            if isinstance(e, dict): +                target_json['build_targets'].append([e['keyboard'], e['keymap']]) +            elif isinstance(e, Path): +                target_json['build_targets'].append(str(e.relative_to(self.path.parent))) + +        try: +            # Ensure what we're writing validates against the latest version of the schema +            validate(target_json, 'qmk.user_repo.v1') +        except jsonschema.ValidationError as err: +            cli.log.error(f'Could not save userspace file: {err}') +            return False + +        # Only actually write out data if it changed +        old_data = json.dumps(json.loads(self.path.read_text()), cls=UserspaceJSONEncoder, sort_keys=True) +        new_data = json.dumps(target_json, cls=UserspaceJSONEncoder, sort_keys=True) +        if old_data != new_data: +            self.path.write_text(new_data) +            cli.log.info(f'Saved userspace file to {self.path}.') +        return True + +    def add_target(self, keyboard=None, keymap=None, json_path=None, do_print=True): +        if json_path is not None: +            # Assume we're adding a json filename/path +            json_path = Path(json_path) +            if json_path not in self.build_targets: +                self.build_targets.append(json_path) +                if do_print: +                    cli.log.info(f'Added {json_path} to userspace build targets.') +            else: +                cli.log.info(f'{json_path} is already a userspace build target.') + +        elif keyboard is not None and keymap is not None: +            # Both keyboard/keymap specified +            e = {"keyboard": keyboard, "keymap": keymap} +            if e not in self.build_targets: +                self.build_targets.append(e) +                if do_print: +                    cli.log.info(f'Added {keyboard}:{keymap} to userspace build targets.') +            else: +                if do_print: +                    cli.log.info(f'{keyboard}:{keymap} is already a userspace build target.') + +    def remove_target(self, keyboard=None, keymap=None, json_path=None, do_print=True): +        if json_path is not None: +            # Assume we're removing a json filename/path +            json_path = Path(json_path) +            if json_path in self.build_targets: +                self.build_targets.remove(json_path) +                if do_print: +                    cli.log.info(f'Removed {json_path} from userspace build targets.') +            else: +                cli.log.info(f'{json_path} is not a userspace build target.') + +        elif keyboard is not None and keymap is not None: +            # Both keyboard/keymap specified +            e = {"keyboard": keyboard, "keymap": keymap} +            if e in self.build_targets: +                self.build_targets.remove(e) +                if do_print: +                    cli.log.info(f'Removed {keyboard}:{keymap} from userspace build targets.') +            else: +                if do_print: +                    cli.log.info(f'{keyboard}:{keymap} is not a userspace build target.') + +    def __load_v1(self, json): +        for e in json['build_targets']: +            if isinstance(e, list) and len(e) == 2: +                self.add_target(keyboard=e[0], keymap=e[1], do_print=False) +            if isinstance(e, str): +                p = self.path.parent / e +                if p.exists() and p.suffix == '.json': +                    self.add_target(json_path=p, do_print=False) + + +class UserspaceValidationError(Exception): +    def __init__(self, *args, **kwargs): +        super().__init__(*args, **kwargs) +        self.__exceptions = [] + +    def __str__(self): +        return self.message + +    @property +    def exceptions(self): +        return self.__exceptions + +    def add(self, schema, exception): +        self.__exceptions.append((schema, exception)) +        errorlist = "\n\n".join([f"{schema}: {exception}" for schema, exception in self.__exceptions]) +        self.message = f'Could not validate against any version of the userspace schema. Errors:\n\n{errorlist}'  | 
