From 3e59bbd7316aaa61a4f90bfa1b50f95cc967d0dc Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Wed, 30 Nov 2022 20:08:54 +0000
Subject: Automate "Data Driven" migrations (#17820)

---
 lib/python/qmk/cli/__init__.py |  1 +
 lib/python/qmk/cli/migrate.py  | 81 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 82 insertions(+)
 create mode 100644 lib/python/qmk/cli/migrate.py

(limited to 'lib/python/qmk')

diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py
index 9190af4e50..4e3ce63da3 100644
--- a/lib/python/qmk/cli/__init__.py
+++ b/lib/python/qmk/cli/__init__.py
@@ -71,6 +71,7 @@ subcommands = [
     'qmk.cli.list.keymaps',
     'qmk.cli.list.layouts',
     'qmk.cli.kle2json',
+    'qmk.cli.migrate',
     'qmk.cli.multibuild',
     'qmk.cli.new.keyboard',
     'qmk.cli.new.keymap',
diff --git a/lib/python/qmk/cli/migrate.py b/lib/python/qmk/cli/migrate.py
new file mode 100644
index 0000000000..4164f9c8ad
--- /dev/null
+++ b/lib/python/qmk/cli/migrate.py
@@ -0,0 +1,81 @@
+"""Migrate keyboard configuration to "Data Driven"
+"""
+import json
+from pathlib import Path
+from dotty_dict import dotty
+
+from milc import cli
+
+from qmk.keyboard import keyboard_completer, keyboard_folder, resolve_keyboard
+from qmk.info import info_json, find_info_json
+from qmk.json_encoders import InfoJSONEncoder
+from qmk.json_schema import json_load
+
+
+def _candidate_files(keyboard):
+    kb_dir = Path(resolve_keyboard(keyboard))
+
+    cur_dir = Path('keyboards')
+    files = []
+    for dir in kb_dir.parts:
+        cur_dir = cur_dir / dir
+        files.append(cur_dir / 'config.h')
+        files.append(cur_dir / 'rules.mk')
+
+    return [file for file in files if file.exists()]
+
+
+@cli.argument('-f', '--filter', arg_only=True, action='append', default=[], help="Filter the performed migrations based on the supplied value. Supported format is 'KEY' located from 'data/mappings'. May be passed multiple times.")
+@cli.argument('-kb', '--keyboard', arg_only=True, type=keyboard_folder, completer=keyboard_completer, required=True, help='The keyboard\'s name')
+@cli.subcommand('Migrate keyboard config to "Data Driven".', hidden=True)
+def migrate(cli):
+    """Migrate keyboard configuration to "Data Driven"
+    """
+    # Merge mappings as we do not care to where "KEY" is found just that its removed
+    info_config_map = json_load(Path('data/mappings/info_config.hjson'))
+    info_rules_map = json_load(Path('data/mappings/info_rules.hjson'))
+    info_map = {**info_config_map, **info_rules_map}
+
+    # Parse target info.json which will receive updates
+    target_info = Path(find_info_json(cli.args.keyboard)[0])
+    info_data = dotty(json_load(target_info))
+
+    # Already parsed used for updates
+    kb_info_json = dotty(info_json(cli.args.keyboard))
+
+    # List of candidate files
+    files = _candidate_files(cli.args.keyboard)
+
+    # Filter down keys if requested
+    keys = info_map.keys()
+    if cli.args.filter:
+        keys = list(set(keys) & set(cli.args.filter))
+
+    cli.log.info(f'{{fg_green}}Migrating keyboard {{fg_cyan}}{cli.args.keyboard}{{fg_green}}.{{fg_reset}}')
+
+    # Start migration
+    for file in files:
+        cli.log.info(f'  Migrating file {file}')
+        file_contents = file.read_text(encoding='utf-8').split('\n')
+        for key in keys:
+            for num, line in enumerate(file_contents):
+                if line.startswith(f'{key} =') or line.startswith(f'#define {key} '):
+                    cli.log.info(f'    Migrating {key}...')
+
+                    while line.rstrip().endswith('\\'):
+                        file_contents.pop(num)
+                        line = file_contents[num]
+                    file_contents.pop(num)
+
+                    update_key = info_map[key]["info_key"]
+                    if update_key in kb_info_json:
+                        info_data[update_key] = kb_info_json[update_key]
+
+        file.write_text('\n'.join(file_contents), encoding='utf-8')
+
+    # Finally write out updated info.json
+    cli.log.info(f'  Updating {target_info}')
+    target_info.write_text(json.dumps(info_data.to_dict(), cls=InfoJSONEncoder))
+
+    cli.log.info(f'{{fg_green}}Migration of keyboard {{fg_cyan}}{cli.args.keyboard}{{fg_green}} complete!{{fg_reset}}')
+    cli.log.info(f"Verify build with {{fg_yellow}}qmk compile -kb {cli.args.keyboard} -km default{{fg_reset}}.")
-- 
cgit v1.2.3


From 82760bcea65d1f9e8f39aaca06f1d3b4e22a1f64 Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Sat, 3 Dec 2022 00:42:54 +0000
Subject: Apply suggested workaround for #18371 (#19226)

Fixes undefined
---
 lib/python/qmk/submodules.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/python/qmk')

diff --git a/lib/python/qmk/submodules.py b/lib/python/qmk/submodules.py
index 52efa602a0..d37ca5e644 100644
--- a/lib/python/qmk/submodules.py
+++ b/lib/python/qmk/submodules.py
@@ -40,7 +40,7 @@ def status():
         else:
             raise ValueError('Unknown `git submodule status` sha-1 prefix character: "%s"' % status)
 
-    submodule_logs = cli.run(['git', 'submodule', '-q', 'foreach', 'git --no-pager log --pretty=format:"$sm_path%x01%h%x01%ad%x01%s%x0A" --date=iso -n1'])
+    submodule_logs = cli.run(['git', 'submodule', '-q', 'foreach', 'git --no-pager log --no-show-signature --pretty=format:"$sm_path%x01%h%x01%ad%x01%s%x0A" --date=iso -n1'])
     for log_line in submodule_logs.stdout.split('\n'):
         if not log_line:
             continue
-- 
cgit v1.2.3


From 32dabd5320b2914bb5631bd6f95b8cd7ca4419e7 Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Sat, 3 Dec 2022 12:04:06 +0000
Subject: Align new-keymap with new-keyboard (#19229)

---
 lib/python/qmk/cli/new/keymap.py | 56 ++++++++++++++++++++++++++++------------
 1 file changed, 39 insertions(+), 17 deletions(-)

(limited to 'lib/python/qmk')

diff --git a/lib/python/qmk/cli/new/keymap.py b/lib/python/qmk/cli/new/keymap.py
index 60cb743cb6..e7823bc46d 100755
--- a/lib/python/qmk/cli/new/keymap.py
+++ b/lib/python/qmk/cli/new/keymap.py
@@ -1,12 +1,32 @@
 """This script automates the copying of the default keymap into your own keymap.
 """
 import shutil
-from pathlib import Path
 
-import qmk.path
+from milc import cli
+from milc.questions import question
+
+from qmk.path import is_keyboard, keymap
+from qmk.git import git_get_username
 from qmk.decorators import automagic_keyboard, automagic_keymap
 from qmk.keyboard import keyboard_completer, keyboard_folder
-from milc import cli
+
+
+def prompt_keyboard():
+    prompt = """{fg_yellow}Select Keyboard{style_reset_all}
+If you`re unsure you can view a full list of supported keyboards with {fg_yellow}qmk list-keyboards{style_reset_all}.
+
+Keyboard Name? """
+
+    return question(prompt)
+
+
+def prompt_user():
+    prompt = """
+{fg_yellow}Name Your Keymap{style_reset_all}
+Used for maintainer, copyright, etc
+
+Your GitHub Username? """
+    return question(prompt, default=git_get_username())
 
 
 @cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='Specify keyboard name. Example: 1upkeyboards/1up60hse')
@@ -17,32 +37,34 @@ from milc import cli
 def new_keymap(cli):
     """Creates a new keymap for the keyboard of your choosing.
     """
-    # ask for user input if keyboard or keymap was not provided in the command line
-    keyboard = cli.config.new_keymap.keyboard if cli.config.new_keymap.keyboard else input("Keyboard Name: ")
-    keymap = cli.config.new_keymap.keymap if cli.config.new_keymap.keymap else input("Keymap Name: ")
+    cli.log.info('{style_bright}Generating a new keymap{style_normal}')
+    cli.echo('')
 
-    # generate keymap paths
-    kb_path = Path('keyboards') / keyboard
-    keymap_path = qmk.path.keymap(keyboard)
-    keymap_path_default = keymap_path / 'default'
-    keymap_path_new = keymap_path / keymap
+    # ask for user input if keyboard or keymap was not provided in the command line
+    kb_name = cli.config.new_keymap.keyboard if cli.config.new_keymap.keyboard else prompt_keyboard()
+    user_name = cli.config.new_keymap.keymap if cli.config.new_keymap.keymap else prompt_user()
 
     # check directories
-    if not kb_path.exists():
-        cli.log.error('Keyboard %s does not exist!', kb_path)
+    if not is_keyboard(kb_name):
+        cli.log.error(f'Keyboard {{fg_cyan}}{kb_name}{{fg_reset}} does not exist! Please choose a valid name.')
         return False
 
+    # generate keymap paths
+    km_path = keymap(kb_name)
+    keymap_path_default = km_path / 'default'
+    keymap_path_new = km_path / user_name
+
     if not keymap_path_default.exists():
-        cli.log.error('Keyboard default %s does not exist!', keymap_path_default)
+        cli.log.error(f'Default keymap {{fg_cyan}}{keymap_path_default}{{fg_reset}} does not exist!')
         return False
 
     if keymap_path_new.exists():
-        cli.log.error('Keymap %s already exists!', keymap_path_new)
+        cli.log.error(f'Keymap {{fg_cyan}}{user_name}{{fg_reset}} already exists! Please choose a different name.')
         return False
 
     # create user directory with default keymap files
     shutil.copytree(keymap_path_default, keymap_path_new, symlinks=True)
 
     # end message to user
-    cli.log.info("%s keymap directory created in: %s", keymap, keymap_path_new)
-    cli.log.info("Compile a firmware with your new keymap by typing: \n\n\tqmk compile -kb %s -km %s\n", keyboard, keymap)
+    cli.log.info(f'{{fg_green}}Created a new keymap called {{fg_cyan}}{user_name}{{fg_green}} in: {{fg_cyan}}{keymap_path_new}.{{fg_reset}}')
+    cli.log.info(f"Compile a firmware with your new keymap by typing: {{fg_yellow}}qmk compile -kb {kb_name} -km {user_name}{{fg_reset}}.")
-- 
cgit v1.2.3


From 9bc7e9afbd32e3ed34580f1be9a79e6040158de5 Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Fri, 9 Dec 2022 00:54:52 +0000
Subject: Initial uk+us DD keymap_extras migration (#19031)

---
 lib/python/qmk/cli/generate/api.py      |  9 +++-
 lib/python/qmk/cli/generate/keycodes.py | 54 ++++++++++++++++++++++++
 lib/python/qmk/keycodes.py              | 73 +++++++++++++++++++++++++--------
 3 files changed, 117 insertions(+), 19 deletions(-)

(limited to 'lib/python/qmk')

diff --git a/lib/python/qmk/cli/generate/api.py b/lib/python/qmk/cli/generate/api.py
index 8650a36b84..dd4830f543 100755
--- a/lib/python/qmk/cli/generate/api.py
+++ b/lib/python/qmk/cli/generate/api.py
@@ -11,7 +11,7 @@ from qmk.info import info_json
 from qmk.json_encoders import InfoJSONEncoder
 from qmk.json_schema import json_load
 from qmk.keyboard import find_readme, list_keyboards
-from qmk.keycodes import load_spec, list_versions
+from qmk.keycodes import load_spec, list_versions, list_languages
 
 DATA_PATH = Path('data')
 TEMPLATE_PATH = DATA_PATH / 'templates/api/'
@@ -44,6 +44,13 @@ def _resolve_keycode_specs(output_folder):
         output_file = output_folder / f'constants/keycodes_{version}.json'
         output_file.write_text(json.dumps(overall, indent=4), encoding='utf-8')
 
+    for lang in list_languages():
+        for version in list_versions(lang):
+            overall = load_spec(version, lang)
+
+            output_file = output_folder / f'constants/keycodes_{lang}_{version}.json'
+            output_file.write_text(json.dumps(overall, indent=4), encoding='utf-8')
+
     # Purge files consumed by 'load_spec'
     shutil.rmtree(output_folder / 'constants/keycodes/')
 
diff --git a/lib/python/qmk/cli/generate/keycodes.py b/lib/python/qmk/cli/generate/keycodes.py
index 29b7db3c80..cf80689708 100644
--- a/lib/python/qmk/cli/generate/keycodes.py
+++ b/lib/python/qmk/cli/generate/keycodes.py
@@ -8,6 +8,24 @@ from qmk.path import normpath
 from qmk.keycodes import load_spec
 
 
+def _render_key(key):
+    width = 7
+    if 'S(' in key:
+        width += len('S()')
+    if 'A(' in key:
+        width += len('A()')
+    if 'RCTL(' in key:
+        width += len('RCTL()')
+    if 'ALGR(' in key:
+        width += len('ALGR()')
+    return key.ljust(width)
+
+
+def _render_label(label):
+    label = label.replace("\\", "(backslash)")
+    return label
+
+
 def _generate_ranges(lines, keycodes):
     lines.append('')
     lines.append('enum qk_keycode_ranges {')
@@ -67,6 +85,22 @@ def _generate_helpers(lines, keycodes):
         lines.append(f'#define IS_{ group.upper() }_KEYCODE(code) ((code) >= {lo} && (code) <= {hi})')
 
 
+def _generate_aliases(lines, keycodes):
+    lines.append('')
+    lines.append('// Aliases')
+    for key, value in keycodes["aliases"].items():
+        define = _render_key(value.get("key"))
+        val = _render_key(key)
+        label = _render_label(value.get("label"))
+
+        lines.append(f'#define {define} {val} // {label}')
+
+    lines.append('')
+    for key, value in keycodes["aliases"].items():
+        for alias in value.get("aliases", []):
+            lines.append(f'#define {alias} {value.get("key")}')
+
+
 @cli.argument('-v', '--version', arg_only=True, required=True, help='Version of keycodes to generate.')
 @cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to')
 @cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages")
@@ -86,3 +120,23 @@ def generate_keycodes(cli):
 
     # Show the results
     dump_lines(cli.args.output, keycodes_h_lines, cli.args.quiet)
+
+
+@cli.argument('-v', '--version', arg_only=True, required=True, help='Version of keycodes to generate.')
+@cli.argument('-l', '--lang', arg_only=True, required=True, help='Language of keycodes to generate.')
+@cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to')
+@cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages")
+@cli.subcommand('Used by the make system to generate keymap_{lang}.h from keycodes_{lang}_{version}.json', hidden=True)
+def generate_keycode_extras(cli):
+    """Generates the header file.
+    """
+
+    # Build the header file.
+    keycodes_h_lines = [GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE, '#pragma once', '#include "keymap.h"', '// clang-format off']
+
+    keycodes = load_spec(cli.args.version, cli.args.lang)
+
+    _generate_aliases(keycodes_h_lines, keycodes)
+
+    # Show the results
+    dump_lines(cli.args.output, keycodes_h_lines, cli.args.quiet)
diff --git a/lib/python/qmk/keycodes.py b/lib/python/qmk/keycodes.py
index cf1ee0767a..600163bab9 100644
--- a/lib/python/qmk/keycodes.py
+++ b/lib/python/qmk/keycodes.py
@@ -2,7 +2,42 @@ from pathlib import Path
 
 from qmk.json_schema import deep_update, json_load, validate
 
-CONSTANTS_PATH = Path('data/constants/keycodes/')
+CONSTANTS_PATH = Path('data/constants/')
+KEYCODES_PATH = CONSTANTS_PATH / 'keycodes'
+EXTRAS_PATH = KEYCODES_PATH / 'extras'
+
+
+def _find_versions(path, prefix):
+    ret = []
+    for file in path.glob(f'{prefix}_[0-9].[0-9].[0-9].hjson'):
+        ret.append(file.stem.split('_')[-1])
+
+    ret.sort(reverse=True)
+    return ret
+
+
+def _load_fragments(path, prefix, version):
+    file = path / f'{prefix}_{version}.hjson'
+    if not file.exists():
+        raise ValueError(f'Requested keycode spec ({prefix}:{version}) is invalid!')
+
+    # Load base
+    spec = json_load(file)
+
+    # Merge in fragments
+    fragments = path.glob(f'{prefix}_{version}_*.hjson')
+    for file in fragments:
+        deep_update(spec, json_load(file))
+
+    return spec
+
+
+def _search_path(lang=None):
+    return EXTRAS_PATH if lang else KEYCODES_PATH
+
+
+def _search_prefix(lang=None):
+    return f'keycodes_{lang}' if lang else 'keycodes'
 
 
 def _validate(spec):
@@ -19,26 +54,20 @@ def _validate(spec):
         raise ValueError(f'Keycode spec contains duplicate keycodes! ({",".join(duplicates)})')
 
 
-def load_spec(version):
+def load_spec(version, lang=None):
     """Build keycode data from the requested spec file
     """
     if version == 'latest':
-        version = list_versions()[0]
+        version = list_versions(lang)[0]
 
-    file = CONSTANTS_PATH / f'keycodes_{version}.hjson'
-    if not file.exists():
-        raise ValueError(f'Requested keycode spec ({version}) is invalid!')
+    path = _search_path(lang)
+    prefix = _search_prefix(lang)
 
     # Load base
-    spec = json_load(file)
-
-    # Merge in fragments
-    fragments = CONSTANTS_PATH.glob(f'keycodes_{version}_*.hjson')
-    for file in fragments:
-        deep_update(spec, json_load(file))
+    spec = _load_fragments(path, prefix, version)
 
     # Sort?
-    spec['keycodes'] = dict(sorted(spec['keycodes'].items()))
+    spec['keycodes'] = dict(sorted(spec.get('keycodes', {}).items()))
 
     # Validate?
     _validate(spec)
@@ -46,12 +75,20 @@ def load_spec(version):
     return spec
 
 
-def list_versions():
+def list_versions(lang=None):
     """Return available versions - sorted newest first
     """
-    ret = []
-    for file in CONSTANTS_PATH.glob('keycodes_[0-9].[0-9].[0-9].hjson'):
-        ret.append(file.stem.split('_')[1])
+    path = _search_path(lang)
+    prefix = _search_prefix(lang)
+
+    return _find_versions(path, prefix)
+
+
+def list_languages():
+    """Return available languages
+    """
+    ret = set()
+    for file in EXTRAS_PATH.glob('keycodes_*_[0-9].[0-9].[0-9].hjson'):
+        ret.add(file.stem.split('_')[1])
 
-    ret.sort(reverse=True)
     return ret
-- 
cgit v1.2.3


From 962e4c0e1854b10612bab547c3d842c5f967dd23 Mon Sep 17 00:00:00 2001
From: Stefan Kerkmann <karlk90@pm.me>
Date: Wed, 14 Dec 2022 16:31:08 +0100
Subject: [Test] Reset timer for every unit test and provide timestamps for log
 messages (#17028)

---
 lib/python/qmk/cli/__init__.py                |  1 +
 lib/python/qmk/cli/generate/keycodes_tests.py | 39 +++++++++++++++++++++++++++
 2 files changed, 40 insertions(+)
 create mode 100644 lib/python/qmk/cli/generate/keycodes_tests.py

(limited to 'lib/python/qmk')

diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py
index 4e3ce63da3..a6d53c1cb6 100644
--- a/lib/python/qmk/cli/__init__.py
+++ b/lib/python/qmk/cli/__init__.py
@@ -57,6 +57,7 @@ subcommands = [
     'qmk.cli.generate.keyboard_c',
     'qmk.cli.generate.keyboard_h',
     'qmk.cli.generate.keycodes',
+    'qmk.cli.generate.keycodes_tests',
     'qmk.cli.generate.rgb_breathe_table',
     'qmk.cli.generate.rules_mk',
     'qmk.cli.generate.version_h',
diff --git a/lib/python/qmk/cli/generate/keycodes_tests.py b/lib/python/qmk/cli/generate/keycodes_tests.py
new file mode 100644
index 0000000000..453b4693a7
--- /dev/null
+++ b/lib/python/qmk/cli/generate/keycodes_tests.py
@@ -0,0 +1,39 @@
+"""Used by the make system to generate a keycode lookup table from keycodes_{version}.json
+"""
+from milc import cli
+
+from qmk.constants import GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE
+from qmk.commands import dump_lines
+from qmk.path import normpath
+from qmk.keycodes import load_spec
+
+
+def _generate_defines(lines, keycodes):
+    lines.append('')
+    lines.append('std::map<uint16_t, std::string> KEYCODE_ID_TABLE = {')
+    for key, value in keycodes["keycodes"].items():
+        lines.append(f'    {{{value.get("key")}, "{value.get("key")}"}},')
+    lines.append('};')
+
+
+@cli.argument('-v', '--version', arg_only=True, required=True, help='Version of keycodes to generate.')
+@cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to')
+@cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages")
+@cli.subcommand('Used by the make system to generate a keycode lookup table from keycodes_{version}.json', hidden=True)
+def generate_keycodes_tests(cli):
+    """Generates a keycode to identifier lookup table for unit test output.
+    """
+
+    # Build the keycodes.h file.
+    keycodes_h_lines = [GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE, '// clang-format off']
+    keycodes_h_lines.append('extern "C" {\n#include <keycode.h>\n}')
+    keycodes_h_lines.append('#include <map>')
+    keycodes_h_lines.append('#include <string>')
+    keycodes_h_lines.append('#include <cstdint>')
+
+    keycodes = load_spec(cli.args.version)
+
+    _generate_defines(keycodes_h_lines, keycodes)
+
+    # Show the results
+    dump_lines(cli.args.output, keycodes_h_lines, cli.args.quiet)
-- 
cgit v1.2.3


From e5721bbd37fcb0372fd007caae5a3f2aed060479 Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Wed, 21 Dec 2022 23:35:23 +0000
Subject: Remaining DD keymap_extras migration  (#19110)

* Parse headers to data

* Regen headers from data
---
 lib/python/qmk/cli/generate/keycodes.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

(limited to 'lib/python/qmk')

diff --git a/lib/python/qmk/cli/generate/keycodes.py b/lib/python/qmk/cli/generate/keycodes.py
index cf80689708..2ed84cd589 100644
--- a/lib/python/qmk/cli/generate/keycodes.py
+++ b/lib/python/qmk/cli/generate/keycodes.py
@@ -91,9 +91,10 @@ def _generate_aliases(lines, keycodes):
     for key, value in keycodes["aliases"].items():
         define = _render_key(value.get("key"))
         val = _render_key(key)
-        label = _render_label(value.get("label"))
-
-        lines.append(f'#define {define} {val} // {label}')
+        if 'label' in value:
+            lines.append(f'#define {define} {val} // {_render_label(value.get("label"))}')
+        else:
+            lines.append(f'#define {define} {val}')
 
     lines.append('')
     for key, value in keycodes["aliases"].items():
-- 
cgit v1.2.3


From 003cee00980172c1305bf97b3c5a5801de336866 Mon Sep 17 00:00:00 2001
From: jack <0x6A73@pm.me>
Date: Fri, 23 Dec 2022 11:18:57 -0700
Subject: Validate keyboard name before accepting further input (#19394)

---
 lib/python/qmk/cli/new/keyboard.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

(limited to 'lib/python/qmk')

diff --git a/lib/python/qmk/cli/new/keyboard.py b/lib/python/qmk/cli/new/keyboard.py
index 251ad919dd..cdd3919168 100644
--- a/lib/python/qmk/cli/new/keyboard.py
+++ b/lib/python/qmk/cli/new/keyboard.py
@@ -195,11 +195,6 @@ def new_keyboard(cli):
     cli.echo('')
 
     kb_name = cli.args.keyboard if cli.args.keyboard else prompt_keyboard()
-    user_name = cli.config.new_keyboard.name if cli.config.new_keyboard.name else prompt_user()
-    real_name = cli.args.realname or cli.config.new_keyboard.name if cli.args.realname or cli.config.new_keyboard.name else prompt_name(user_name)
-    default_layout = cli.args.layout if cli.args.layout else prompt_layout()
-    mcu = cli.args.type if cli.args.type else prompt_mcu()
-
     if not validate_keyboard_name(kb_name):
         cli.log.error('Keyboard names must contain only {fg_cyan}lowercase a-z{fg_reset}, {fg_cyan}0-9{fg_reset}, and {fg_cyan}_{fg_reset}! Please choose a different name.')
         return 1
@@ -208,6 +203,11 @@ def new_keyboard(cli):
         cli.log.error(f'Keyboard {{fg_cyan}}{kb_name}{{fg_reset}} already exists! Please choose a different name.')
         return 1
 
+    user_name = cli.config.new_keyboard.name if cli.config.new_keyboard.name else prompt_user()
+    real_name = cli.args.realname or cli.config.new_keyboard.name if cli.args.realname or cli.config.new_keyboard.name else prompt_name(user_name)
+    default_layout = cli.args.layout if cli.args.layout else prompt_layout()
+    mcu = cli.args.type if cli.args.type else prompt_mcu()
+
     # Preprocess any development_board presets
     if mcu in dev_boards:
         defaults_map = json_load(Path('data/mappings/defaults.hjson'))
-- 
cgit v1.2.3


From e4cfbd2532aa173fe8389de5f2935989c0be804d Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Sun, 1 Jan 2023 04:51:29 +0000
Subject: Allow CLI to flash .uf2 files (#19462)

---
 lib/python/qmk/cli/flash.py | 46 ++++++++++++++++++++++++---------------------
 lib/python/qmk/flashers.py  | 14 ++++++++++++++
 2 files changed, 39 insertions(+), 21 deletions(-)

(limited to 'lib/python/qmk')

diff --git a/lib/python/qmk/cli/flash.py b/lib/python/qmk/cli/flash.py
index 40bfbdab56..52defb5f0d 100644
--- a/lib/python/qmk/cli/flash.py
+++ b/lib/python/qmk/cli/flash.py
@@ -11,12 +11,14 @@ import qmk.path
 from qmk.decorators import automagic_keyboard, automagic_keymap
 from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json, build_environment
 from qmk.keyboard import keyboard_completer, keyboard_folder
+from qmk.keymap import keymap_completer
 from qmk.flashers import flasher
 
 
-def print_bootloader_help():
+def _list_bootloaders():
     """Prints the available bootloaders listed in docs.qmk.fm.
     """
+    cli.print_help()
     cli.log.info('Here are the available bootloaders:')
     cli.echo('\tavrdude')
     cli.echo('\tbootloadhid')
@@ -36,14 +38,29 @@ def print_bootloader_help():
     cli.echo('\tuf2-split-left')
     cli.echo('\tuf2-split-right')
     cli.echo('For more info, visit https://docs.qmk.fm/#/flashing')
+    return False
+
+
+def _flash_binary(filename, mcu):
+    """Try to flash binary firmware
+    """
+    cli.echo('Flashing binary firmware...\nPlease reset your keyboard into bootloader mode now!\nPress Ctrl-C to exit.\n')
+    try:
+        err, msg = flasher(mcu, filename)
+        if err:
+            cli.log.error(msg)
+            return False
+    except KeyboardInterrupt:
+        cli.log.info('Ctrl-C was pressed, exiting...')
+    return True
 
 
 @cli.argument('filename', nargs='?', arg_only=True, type=qmk.path.FileType('r'), completer=FilesCompleter('.json'), help='A configurator export JSON to be compiled and flashed or a pre-compiled binary firmware file (bin/hex) to be flashed.')
 @cli.argument('-b', '--bootloaders', action='store_true', help='List the available bootloaders.')
 @cli.argument('-bl', '--bootloader', default='flash', help='The flash command, corresponding to qmk\'s make options of bootloaders.')
 @cli.argument('-m', '--mcu', help='The MCU name. Required for HalfKay, HID, USBAspLoader and ISP flashing.')
-@cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.')
-@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.')
+@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
+@cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.')
 @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.")
 @cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")
 @cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")
@@ -56,30 +73,17 @@ def flash(cli):
 
     If a binary firmware is supplied, try to flash that.
 
-    If a Configurator JSON export is supplied this command will create a new keymap. Keymap and Keyboard arguments
-    will be ignored.
+    If a Configurator export is supplied this command will create a new keymap, overwriting an existing keymap if one exists.
 
-    If no file is supplied, keymap and keyboard are expected.
+    If a keyboard and keymap are provided this command will build a firmware based on that.
 
     If bootloader is omitted the make system will use the configured bootloader for that keyboard.
     """
-    if cli.args.filename and cli.args.filename.suffix in ['.bin', '.hex']:
-        # Try to flash binary firmware
-        cli.echo('Flashing binary firmware...\nPlease reset your keyboard into bootloader mode now!\nPress Ctrl-C to exit.\n')
-        try:
-            err, msg = flasher(cli.args.mcu, cli.args.filename)
-            if err:
-                cli.log.error(msg)
-                return False
-        except KeyboardInterrupt:
-            cli.log.info('Ctrl-C was pressed, exiting...')
-        return True
+    if cli.args.filename and cli.args.filename.suffix in ['.bin', '.hex', '.uf2']:
+        return _flash_binary(cli.args.filename, cli.args.mcu)
 
     if cli.args.bootloaders:
-        # Provide usage and list bootloaders
-        cli.print_help()
-        print_bootloader_help()
-        return False
+        return _list_bootloaders()
 
     # Build the environment vars
     envs = build_environment(cli.args.env)
diff --git a/lib/python/qmk/flashers.py b/lib/python/qmk/flashers.py
index e902e5072f..f83665d9ac 100644
--- a/lib/python/qmk/flashers.py
+++ b/lib/python/qmk/flashers.py
@@ -71,6 +71,12 @@ def _find_usb_device(vid_hex, pid_hex):
             return usb.core.find(idVendor=vid_hex, idProduct=pid_hex)
 
 
+def _find_uf2_devices():
+    """Delegate to uf2conv.py as VID:PID pairs can potentially fluctuate more than other bootloaders
+    """
+    return cli.run(['util/uf2conv.py', '--list']).stdout.splitlines()
+
+
 def _find_bootloader():
     # To avoid running forever in the background, only look for bootloaders for 10min
     start_time = time.time()
@@ -95,6 +101,8 @@ def _find_bootloader():
                     else:
                         details = None
                     return (bl, details)
+        if _find_uf2_devices():
+            return ('_uf2_compatible_', None)
         time.sleep(0.1)
     return (None, None)
 
@@ -184,6 +192,10 @@ def _flash_mdloader(file):
     cli.run(['mdloader', '--first', '--download', file, '--restart'], capture_output=False)
 
 
+def _flash_uf2(file):
+    cli.run(['util/uf2conv.py', '--deploy', file], capture_output=False)
+
+
 def flasher(mcu, file):
     bl, details = _find_bootloader()
     # Add a small sleep to avoid race conditions
@@ -208,6 +220,8 @@ def flasher(mcu, file):
             return (True, "Specifying the MCU with '-m' is necessary for ISP flashing!")
     elif bl == 'md-boot':
         _flash_mdloader(file)
+    elif bl == '_uf2_compatible_':
+        _flash_uf2(file)
     else:
         return (True, "Known bootloader found but flashing not currently supported!")
 
-- 
cgit v1.2.3


From 24adecd9227931ebaec16115e05055e0b580d351 Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Sun, 1 Jan 2023 19:16:38 +0000
Subject: Implement XAP style merge semantics for DD keycodes (#19397)

---
 lib/python/qmk/json_schema.py | 36 ++++++++++++++++++++++++++++--
 lib/python/qmk/keycodes.py    | 52 ++++++++++++++++++++++++++++++-------------
 2 files changed, 71 insertions(+), 17 deletions(-)

(limited to 'lib/python/qmk')

diff --git a/lib/python/qmk/json_schema.py b/lib/python/qmk/json_schema.py
index 934e2f841f..c886a0d868 100644
--- a/lib/python/qmk/json_schema.py
+++ b/lib/python/qmk/json_schema.py
@@ -1,12 +1,13 @@
 """Functions that help us generate and use info.json files.
 """
 import json
+import hjson
+import jsonschema
 from collections.abc import Mapping
 from functools import lru_cache
+from typing import OrderedDict
 from pathlib import Path
 
-import hjson
-import jsonschema
 from milc import cli
 
 
@@ -101,3 +102,34 @@ def deep_update(origdict, newdict):
             origdict[key] = value
 
     return origdict
+
+
+def merge_ordered_dicts(dicts):
+    """Merges nested OrderedDict objects resulting from reading a hjson file.
+    Later input dicts overrides earlier dicts for plain values.
+    Arrays will be appended. If the first entry of an array is "!reset!", the contents of the array will be cleared and replaced with RHS.
+    Dictionaries will be recursively merged. If any entry is "!reset!", the contents of the dictionary will be cleared and replaced with RHS.
+    """
+    result = OrderedDict()
+
+    def add_entry(target, k, v):
+        if k in target and isinstance(v, (OrderedDict, dict)):
+            if "!reset!" in v:
+                target[k] = v
+            else:
+                target[k] = merge_ordered_dicts([target[k], v])
+            if "!reset!" in target[k]:
+                del target[k]["!reset!"]
+        elif k in target and isinstance(v, list):
+            if v[0] == '!reset!':
+                target[k] = v[1:]
+            else:
+                target[k] = target[k] + v
+        else:
+            target[k] = v
+
+    for d in dicts:
+        for (k, v) in d.items():
+            add_entry(result, k, v)
+
+    return result
diff --git a/lib/python/qmk/keycodes.py b/lib/python/qmk/keycodes.py
index 600163bab9..d2f2492829 100644
--- a/lib/python/qmk/keycodes.py
+++ b/lib/python/qmk/keycodes.py
@@ -1,6 +1,6 @@
 from pathlib import Path
 
-from qmk.json_schema import deep_update, json_load, validate
+from qmk.json_schema import merge_ordered_dicts, deep_update, json_load, validate
 
 CONSTANTS_PATH = Path('data/constants/')
 KEYCODES_PATH = CONSTANTS_PATH / 'keycodes'
@@ -16,20 +16,13 @@ def _find_versions(path, prefix):
     return ret
 
 
-def _load_fragments(path, prefix, version):
-    file = path / f'{prefix}_{version}.hjson'
-    if not file.exists():
-        raise ValueError(f'Requested keycode spec ({prefix}:{version}) is invalid!')
+def _potential_search_versions(version, lang=None):
+    versions = list_versions(lang)
+    versions.reverse()
 
-    # Load base
-    spec = json_load(file)
+    loc = versions.index(version) + 1
 
-    # Merge in fragments
-    fragments = path.glob(f'{prefix}_{version}_*.hjson')
-    for file in fragments:
-        deep_update(spec, json_load(file))
-
-    return spec
+    return versions[:loc]
 
 
 def _search_path(lang=None):
@@ -40,6 +33,34 @@ def _search_prefix(lang=None):
     return f'keycodes_{lang}' if lang else 'keycodes'
 
 
+def _locate_files(path, prefix, versions):
+    # collate files by fragment "type"
+    files = {'_': []}
+    for version in versions:
+        files['_'].append(path / f'{prefix}_{version}.hjson')
+
+        for file in path.glob(f'{prefix}_{version}_*.hjson'):
+            fragment = file.stem.replace(f'{prefix}_{version}_', '')
+            if fragment not in files:
+                files[fragment] = []
+            files[fragment].append(file)
+
+    return files
+
+
+def _process_files(files):
+    # allow override within types of fragments - but not globally
+    spec = {}
+    for category in files.values():
+        specs = []
+        for file in category:
+            specs.append(json_load(file))
+
+        deep_update(spec, merge_ordered_dicts(specs))
+
+    return spec
+
+
 def _validate(spec):
     # first throw it to the jsonschema
     validate(spec, 'qmk.keycodes.v1')
@@ -62,9 +83,10 @@ def load_spec(version, lang=None):
 
     path = _search_path(lang)
     prefix = _search_prefix(lang)
+    versions = _potential_search_versions(version, lang)
 
-    # Load base
-    spec = _load_fragments(path, prefix, version)
+    # Load bases + any fragments
+    spec = _process_files(_locate_files(path, prefix, versions))
 
     # Sort?
     spec['keycodes'] = dict(sorted(spec.get('keycodes', {}).items()))
-- 
cgit v1.2.3


From 3a5a4c708f4817919d9fe7396ceb2b8e57d2e68e Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Mon, 2 Jan 2023 22:00:29 +0000
Subject: Report submodule status when not valid work-tree (#19474)

---
 lib/python/qmk/cli/doctor/check.py |  2 --
 lib/python/qmk/cli/doctor/main.py  |  2 +-
 lib/python/qmk/submodules.py       | 24 ++++++++++--------------
 3 files changed, 11 insertions(+), 17 deletions(-)

(limited to 'lib/python/qmk')

diff --git a/lib/python/qmk/cli/doctor/check.py b/lib/python/qmk/cli/doctor/check.py
index 8a0422ba72..a368242fdb 100644
--- a/lib/python/qmk/cli/doctor/check.py
+++ b/lib/python/qmk/cli/doctor/check.py
@@ -119,10 +119,8 @@ def check_submodules():
     """
     for submodule in submodules.status().values():
         if submodule['status'] is None:
-            cli.log.error('Submodule %s has not yet been cloned!', submodule['name'])
             return CheckStatus.ERROR
         elif not submodule['status']:
-            cli.log.warning('Submodule %s is not up to date!', submodule['name'])
             return CheckStatus.WARNING
 
     return CheckStatus.OK
diff --git a/lib/python/qmk/cli/doctor/main.py b/lib/python/qmk/cli/doctor/main.py
index 1600ab8dd4..d55a11e5fd 100755
--- a/lib/python/qmk/cli/doctor/main.py
+++ b/lib/python/qmk/cli/doctor/main.py
@@ -142,7 +142,7 @@ def doctor(cli):
     if sub_ok == CheckStatus.OK:
         cli.log.info('Submodules are up to date.')
     else:
-        if yesno('Would you like to clone the submodules?', default=True):
+        if git_check_repo() and yesno('Would you like to clone the submodules?', default=True):
             submodules.update()
             sub_ok = check_submodules()
 
diff --git a/lib/python/qmk/submodules.py b/lib/python/qmk/submodules.py
index d37ca5e644..d0050b371d 100644
--- a/lib/python/qmk/submodules.py
+++ b/lib/python/qmk/submodules.py
@@ -21,15 +21,17 @@ def status():
     status is None when the submodule doesn't exist, False when it's out of date, and True when it's current
     """
     submodules = {}
-    git_cmd = cli.run(['git', 'submodule', 'status'], timeout=30)
-
-    for line in git_cmd.stdout.split('\n'):
-        if not line:
-            continue
+    gitmodule_config = cli.run(['git', 'config', '-f', '.gitmodules', '-l'], timeout=30)
+    for line in gitmodule_config.stdout.splitlines():
+        key, value = line.split('=', maxsplit=2)
+        if key.endswith('.path'):
+            submodules[value] = {'name': value, 'status': None}
 
+    git_cmd = cli.run(['git', 'submodule', 'status'], timeout=30)
+    for line in git_cmd.stdout.splitlines():
         status = line[0]
         githash, submodule = line[1:].split()[:2]
-        submodules[submodule] = {'name': submodule, 'githash': githash}
+        submodules[submodule]['githash'] = githash
 
         if status == '-':
             submodules[submodule]['status'] = None
@@ -41,10 +43,7 @@ def status():
             raise ValueError('Unknown `git submodule status` sha-1 prefix character: "%s"' % status)
 
     submodule_logs = cli.run(['git', 'submodule', '-q', 'foreach', 'git --no-pager log --no-show-signature --pretty=format:"$sm_path%x01%h%x01%ad%x01%s%x0A" --date=iso -n1'])
-    for log_line in submodule_logs.stdout.split('\n'):
-        if not log_line:
-            continue
-
+    for log_line in submodule_logs.stdout.splitlines():
         r = log_line.split('\x01')
         submodule = r[0]
         submodules[submodule]['shorthash'] = r[1] if len(r) > 1 else ''
@@ -52,10 +51,7 @@ def status():
         submodules[submodule]['last_log_message'] = r[3] if len(r) > 3 else ''
 
     submodule_tags = cli.run(['git', 'submodule', '-q', 'foreach', '\'echo $sm_path `git describe --tags`\''])
-    for log_line in submodule_tags.stdout.split('\n'):
-        if not log_line:
-            continue
-
+    for log_line in submodule_tags.stdout.splitlines():
         r = log_line.split()
         submodule = r[0]
         submodules[submodule]['describe'] = r[1] if len(r) > 1 else ''
-- 
cgit v1.2.3


From b297531dbf645a28fd5a67e546a9ff27d49e1b1e Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Mon, 2 Jan 2023 22:11:57 +0000
Subject: Migrate 'make git-submodule' to CLI command (#19479)

---
 lib/python/qmk/cli/__init__.py      |  1 +
 lib/python/qmk/cli/git/__init__.py  |  0
 lib/python/qmk/cli/git/submodule.py | 22 ++++++++++++++++++++++
 3 files changed, 23 insertions(+)
 create mode 100644 lib/python/qmk/cli/git/__init__.py
 create mode 100644 lib/python/qmk/cli/git/submodule.py

(limited to 'lib/python/qmk')

diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py
index a6d53c1cb6..e0ab67da7b 100644
--- a/lib/python/qmk/cli/__init__.py
+++ b/lib/python/qmk/cli/__init__.py
@@ -61,6 +61,7 @@ subcommands = [
     'qmk.cli.generate.rgb_breathe_table',
     'qmk.cli.generate.rules_mk',
     'qmk.cli.generate.version_h',
+    'qmk.cli.git.submodule',
     'qmk.cli.hello',
     'qmk.cli.import.kbfirmware',
     'qmk.cli.import.keyboard',
diff --git a/lib/python/qmk/cli/git/__init__.py b/lib/python/qmk/cli/git/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/lib/python/qmk/cli/git/submodule.py b/lib/python/qmk/cli/git/submodule.py
new file mode 100644
index 0000000000..bcc3868a52
--- /dev/null
+++ b/lib/python/qmk/cli/git/submodule.py
@@ -0,0 +1,22 @@
+import shutil
+from qmk.path import normpath
+
+from milc import cli
+
+REMOVE_DIRS = [
+    'lib/ugfx',
+    'lib/pico-sdk',
+    'lib/chibios-contrib/ext/mcux-sdk',
+    'lib/lvgl',
+]
+
+
+@cli.subcommand('Git Submodule actions.')
+def git_submodule(cli):
+    for folder in REMOVE_DIRS:
+        if normpath(folder).is_dir():
+            print(f"Removing '{folder}'")
+            shutil.rmtree(folder)
+
+    cli.run(['git', 'submodule', 'sync', '--recursive'], capture_output=False)
+    cli.run(['git', 'submodule', 'update', '--init', '--recursive', '--progress'], capture_output=False)
-- 
cgit v1.2.3


From c345278101b3882a2f33b078021ab31a6129120a Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Tue, 3 Jan 2023 03:15:29 +0000
Subject: Replace list_keyboards.sh with CLI calls (#19485)

---
 lib/python/qmk/cli/list/keyboards.py |  3 ++-
 lib/python/qmk/keyboard.py           | 10 +++++++---
 2 files changed, 9 insertions(+), 4 deletions(-)

(limited to 'lib/python/qmk')

diff --git a/lib/python/qmk/cli/list/keyboards.py b/lib/python/qmk/cli/list/keyboards.py
index 8b6c451673..405b9210e4 100644
--- a/lib/python/qmk/cli/list/keyboards.py
+++ b/lib/python/qmk/cli/list/keyboards.py
@@ -5,9 +5,10 @@ from milc import cli
 import qmk.keyboard
 
 
+@cli.argument('--no-resolve-defaults', arg_only=True, action='store_false', help='Ignore any "DEFAULT_FOLDER" within keyboards rules.mk')
 @cli.subcommand("List the keyboards currently defined within QMK")
 def list_keyboards(cli):
     """List the keyboards currently defined within QMK
     """
-    for keyboard_name in qmk.keyboard.list_keyboards():
+    for keyboard_name in qmk.keyboard.list_keyboards(cli.args.no_resolve_defaults):
         print(keyboard_name)
diff --git a/lib/python/qmk/keyboard.py b/lib/python/qmk/keyboard.py
index 6ddbba8fa5..0c980faf2b 100644
--- a/lib/python/qmk/keyboard.py
+++ b/lib/python/qmk/keyboard.py
@@ -98,14 +98,18 @@ def keyboard_completer(prefix, action, parser, parsed_args):
     return list_keyboards()
 
 
-def list_keyboards():
-    """Returns a list of all keyboards.
+def list_keyboards(resolve_defaults=True):
+    """Returns a list of all keyboards - optionally processing any DEFAULT_FOLDER.
     """
     # We avoid pathlib here because this is performance critical code.
     kb_wildcard = os.path.join(base_path, "**", "rules.mk")
     paths = [path for path in glob(kb_wildcard, recursive=True) if os.path.sep + 'keymaps' + os.path.sep not in path]
 
-    return sorted(set(map(resolve_keyboard, map(_find_name, paths))))
+    found = map(_find_name, paths)
+    if resolve_defaults:
+        found = map(resolve_keyboard, found)
+
+    return sorted(set(found))
 
 
 def resolve_keyboard(keyboard):
-- 
cgit v1.2.3


From 5c730d971ec78344e1304f746436ac0e1d61aea5 Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Fri, 6 Jan 2023 04:16:52 +0000
Subject: Migrate submodule dirty check to CLI (#19488)

---
 lib/python/qmk/cli/git/submodule.py | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

(limited to 'lib/python/qmk')

diff --git a/lib/python/qmk/cli/git/submodule.py b/lib/python/qmk/cli/git/submodule.py
index bcc3868a52..9f354c021e 100644
--- a/lib/python/qmk/cli/git/submodule.py
+++ b/lib/python/qmk/cli/git/submodule.py
@@ -1,8 +1,10 @@
 import shutil
-from qmk.path import normpath
 
 from milc import cli
 
+from qmk.path import normpath
+from qmk import submodules
+
 REMOVE_DIRS = [
     'lib/ugfx',
     'lib/pico-sdk',
@@ -11,8 +13,22 @@ REMOVE_DIRS = [
 ]
 
 
+@cli.argument('--check', arg_only=True, action='store_true', help='Check if the submodules are dirty, and display a warning if they are.')
+@cli.argument('--sync', arg_only=True, action='store_true', help='Shallow clone any missing submodules.')
 @cli.subcommand('Git Submodule actions.')
 def git_submodule(cli):
+    """Git Submodule actions
+    """
+    if cli.args.check:
+        return all(item['status'] for item in submodules.status().values())
+
+    if cli.args.sync:
+        cli.run(['git', 'submodule', 'sync', '--recursive'])
+        for name, item in submodules.status().items():
+            if item['status'] is None:
+                cli.run(['git', 'submodule', 'update', '--depth=50', '--init', name], capture_output=False)
+        return True
+
     for folder in REMOVE_DIRS:
         if normpath(folder).is_dir():
             print(f"Removing '{folder}'")
-- 
cgit v1.2.3


From 974a1eaf2a1076de0cc06deeefe12e15b7e209bc Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Sat, 7 Jan 2023 17:05:53 +0000
Subject: Ignore defaults.hjson values if already set (#19511)

* Ignore defaults.hjson values if already set

* Add warning when nothing is merged
---
 lib/python/qmk/info.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

(limited to 'lib/python/qmk')

diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py
index 7e588b5182..86b1e2c063 100644
--- a/lib/python/qmk/info.py
+++ b/lib/python/qmk/info.py
@@ -561,8 +561,16 @@ def _process_defaults(info_data):
     for default_type in defaults_map.keys():
         thing_map = defaults_map[default_type]
         if default_type in info_data:
-            for key, value in thing_map.get(info_data[default_type], {}).items():
-                info_data[key] = value
+            merged_count = 0
+            thing_items = thing_map.get(info_data[default_type], {}).items()
+            for key, value in thing_items:
+                if key not in info_data:
+                    info_data[key] = value
+                    merged_count += 1
+
+            if merged_count == 0 and len(thing_items) > 0:
+                _log_warning(info_data, 'All defaults for \'%s\' were skipped, potential redundant config or misconfiguration detected' % (default_type))
+
     return info_data
 
 
-- 
cgit v1.2.3


From 1b045b1e60c2c10178828b20036a3590784bbbfc Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Mon, 9 Jan 2023 08:21:21 +0000
Subject: Handle doctor permission issues while checking udev (#19548)

---
 lib/python/qmk/cli/doctor/linux.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

(limited to 'lib/python/qmk')

diff --git a/lib/python/qmk/cli/doctor/linux.py b/lib/python/qmk/cli/doctor/linux.py
index a803305c0d..95bafe8c64 100644
--- a/lib/python/qmk/cli/doctor/linux.py
+++ b/lib/python/qmk/cli/doctor/linux.py
@@ -78,10 +78,13 @@ def check_udev_rules():
 
         # Collect all rules from the config files
         for rule_file in udev_rules:
-            for line in rule_file.read_text(encoding='utf-8').split('\n'):
-                line = line.strip()
-                if not line.startswith("#") and len(line):
-                    current_rules.add(line)
+            try:
+                for line in rule_file.read_text(encoding='utf-8').split('\n'):
+                    line = line.strip()
+                    if not line.startswith("#") and len(line):
+                        current_rules.add(line)
+            except PermissionError:
+                cli.log.debug("Failed to read: %s", rule_file)
 
         # Check if the desired rules are among the currently present rules
         for bootloader, rules in desired_rules.items():
-- 
cgit v1.2.3


From b57714f793058cfef3e10c0bbdc9ffb02b20c5a7 Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Mon, 9 Jan 2023 09:27:41 +0000
Subject: `qmk doctor` - Handle timeouts while checking binaries (#19549)

---
 lib/python/qmk/cli/doctor/check.py | 73 ++++++++++++++++++++++----------------
 lib/python/qmk/cli/doctor/main.py  |  6 ++--
 2 files changed, 46 insertions(+), 33 deletions(-)

(limited to 'lib/python/qmk')

diff --git a/lib/python/qmk/cli/doctor/check.py b/lib/python/qmk/cli/doctor/check.py
index a368242fdb..426876e98a 100644
--- a/lib/python/qmk/cli/doctor/check.py
+++ b/lib/python/qmk/cli/doctor/check.py
@@ -3,7 +3,7 @@
 from enum import Enum
 import re
 import shutil
-from subprocess import DEVNULL
+from subprocess import DEVNULL, TimeoutExpired
 
 from milc import cli
 from qmk import submodules
@@ -41,9 +41,8 @@ def _parse_gcc_version(version):
 def _check_arm_gcc_version():
     """Returns True if the arm-none-eabi-gcc version is not known to cause problems.
     """
-    if 'output' in ESSENTIAL_BINARIES['arm-none-eabi-gcc']:
-        version_number = ESSENTIAL_BINARIES['arm-none-eabi-gcc']['output'].strip()
-        cli.log.info('Found arm-none-eabi-gcc version %s', version_number)
+    version_number = ESSENTIAL_BINARIES['arm-none-eabi-gcc']['output'].strip()
+    cli.log.info('Found arm-none-eabi-gcc version %s', version_number)
 
     return CheckStatus.OK  # Right now all known arm versions are ok
 
@@ -51,44 +50,37 @@ def _check_arm_gcc_version():
 def _check_avr_gcc_version():
     """Returns True if the avr-gcc version is not known to cause problems.
     """
-    rc = CheckStatus.ERROR
-    if 'output' in ESSENTIAL_BINARIES['avr-gcc']:
-        version_number = ESSENTIAL_BINARIES['avr-gcc']['output'].strip()
+    version_number = ESSENTIAL_BINARIES['avr-gcc']['output'].strip()
+    cli.log.info('Found avr-gcc version %s', version_number)
 
-        cli.log.info('Found avr-gcc version %s', version_number)
-        rc = CheckStatus.OK
+    parsed_version = _parse_gcc_version(version_number)
+    if parsed_version['major'] > 8:
+        cli.log.warning('{fg_yellow}We do not recommend avr-gcc newer than 8. Downgrading to 8.x is recommended.')
+        return CheckStatus.WARNING
 
-        parsed_version = _parse_gcc_version(version_number)
-        if parsed_version['major'] > 8:
-            cli.log.warning('{fg_yellow}We do not recommend avr-gcc newer than 8. Downgrading to 8.x is recommended.')
-            rc = CheckStatus.WARNING
-
-    return rc
+    return CheckStatus.OK
 
 
 def _check_avrdude_version():
-    if 'output' in ESSENTIAL_BINARIES['avrdude']:
-        last_line = ESSENTIAL_BINARIES['avrdude']['output'].split('\n')[-2]
-        version_number = last_line.split()[2][:-1]
-        cli.log.info('Found avrdude version %s', version_number)
+    last_line = ESSENTIAL_BINARIES['avrdude']['output'].split('\n')[-2]
+    version_number = last_line.split()[2][:-1]
+    cli.log.info('Found avrdude version %s', version_number)
 
     return CheckStatus.OK
 
 
 def _check_dfu_util_version():
-    if 'output' in ESSENTIAL_BINARIES['dfu-util']:
-        first_line = ESSENTIAL_BINARIES['dfu-util']['output'].split('\n')[0]
-        version_number = first_line.split()[1]
-        cli.log.info('Found dfu-util version %s', version_number)
+    first_line = ESSENTIAL_BINARIES['dfu-util']['output'].split('\n')[0]
+    version_number = first_line.split()[1]
+    cli.log.info('Found dfu-util version %s', version_number)
 
     return CheckStatus.OK
 
 
 def _check_dfu_programmer_version():
-    if 'output' in ESSENTIAL_BINARIES['dfu-programmer']:
-        first_line = ESSENTIAL_BINARIES['dfu-programmer']['output'].split('\n')[0]
-        version_number = first_line.split()[1]
-        cli.log.info('Found dfu-programmer version %s', version_number)
+    first_line = ESSENTIAL_BINARIES['dfu-programmer']['output'].split('\n')[0]
+    version_number = first_line.split()[1]
+    cli.log.info('Found dfu-programmer version %s', version_number)
 
     return CheckStatus.OK
 
@@ -96,11 +88,16 @@ def _check_dfu_programmer_version():
 def check_binaries():
     """Iterates through ESSENTIAL_BINARIES and tests them.
     """
-    ok = True
+    ok = CheckStatus.OK
 
     for binary in sorted(ESSENTIAL_BINARIES):
-        if not is_executable(binary):
-            ok = False
+        try:
+            if not is_executable(binary):
+                ok = CheckStatus.ERROR
+        except TimeoutExpired:
+            cli.log.debug('Timeout checking %s', binary)
+            if ok != CheckStatus.ERROR:
+                ok = CheckStatus.WARNING
 
     return ok
 
@@ -108,8 +105,22 @@ def check_binaries():
 def check_binary_versions():
     """Check the versions of ESSENTIAL_BINARIES
     """
+    checks = {
+        'arm-none-eabi-gcc': _check_arm_gcc_version,
+        'avr-gcc': _check_avr_gcc_version,
+        'avrdude': _check_avrdude_version,
+        'dfu-util': _check_dfu_util_version,
+        'dfu-programmer': _check_dfu_programmer_version,
+    }
+
     versions = []
-    for check in (_check_arm_gcc_version, _check_avr_gcc_version, _check_avrdude_version, _check_dfu_util_version, _check_dfu_programmer_version):
+    for binary in sorted(ESSENTIAL_BINARIES):
+        if 'output' not in ESSENTIAL_BINARIES[binary]:
+            cli.log.warning('Unknown version for %s', binary)
+            versions.append(CheckStatus.WARNING)
+            continue
+
+        check = checks[binary]
         versions.append(check())
     return versions
 
diff --git a/lib/python/qmk/cli/doctor/main.py b/lib/python/qmk/cli/doctor/main.py
index d55a11e5fd..6a6feb87d1 100755
--- a/lib/python/qmk/cli/doctor/main.py
+++ b/lib/python/qmk/cli/doctor/main.py
@@ -119,13 +119,15 @@ def doctor(cli):
     # Make sure the basic CLI tools we need are available and can be executed.
     bin_ok = check_binaries()
 
-    if not bin_ok:
+    if bin_ok == CheckStatus.ERROR:
         if yesno('Would you like to install dependencies?', default=True):
             cli.run(['util/qmk_install.sh', '-y'], stdin=DEVNULL, capture_output=False)
             bin_ok = check_binaries()
 
-    if bin_ok:
+    if bin_ok == CheckStatus.OK:
         cli.log.info('All dependencies are installed.')
+    elif bin_ok == CheckStatus.WARNING:
+        cli.log.warning('Issues encountered while checking dependencies.')
     else:
         status = CheckStatus.ERROR
 
-- 
cgit v1.2.3


From 20474ae2321f0fb456731646b5b2e3090990b8df Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Wed, 11 Jan 2023 01:38:35 +0000
Subject: Fix CLI community detection (#19562)

---
 lib/python/qmk/cli/generate/api.py |  6 ++++
 lib/python/qmk/info.py             | 11 ++++----
 lib/python/qmk/keymap.py           | 56 +++++++++++++++++++-------------------
 3 files changed, 40 insertions(+), 33 deletions(-)

(limited to 'lib/python/qmk')

diff --git a/lib/python/qmk/cli/generate/api.py b/lib/python/qmk/cli/generate/api.py
index dd4830f543..d662e14e00 100755
--- a/lib/python/qmk/cli/generate/api.py
+++ b/lib/python/qmk/cli/generate/api.py
@@ -10,6 +10,7 @@ from qmk.datetime import current_datetime
 from qmk.info import info_json
 from qmk.json_encoders import InfoJSONEncoder
 from qmk.json_schema import json_load
+from qmk.keymap import list_keymaps
 from qmk.keyboard import find_readme, list_keyboards
 from qmk.keycodes import load_spec, list_versions, list_languages
 
@@ -111,6 +112,11 @@ def generate_api(cli):
     # Generate and write keyboard specific JSON files
     for keyboard_name in keyboard_list:
         kb_all[keyboard_name] = info_json(keyboard_name)
+
+        # Populate the list of JSON keymaps
+        for keymap in list_keymaps(keyboard_name, c=False, fullpath=True):
+            kb_all[keyboard_name]['keymaps'][keymap.name] = {'url': f'https://raw.githubusercontent.com/qmk/qmk_firmware/master/{keymap}/keymap.json'}
+
         keyboard_dir = v1_dir / 'keyboards' / keyboard_name
         keyboard_info = keyboard_dir / 'info.json'
         keyboard_readme = keyboard_dir / 'readme.md'
diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py
index 86b1e2c063..af3b0f3091 100644
--- a/lib/python/qmk/info.py
+++ b/lib/python/qmk/info.py
@@ -10,7 +10,6 @@ from qmk.constants import CHIBIOS_PROCESSORS, LUFA_PROCESSORS, VUSB_PROCESSORS
 from qmk.c_parse import find_layouts, parse_config_h_file, find_led_config
 from qmk.json_schema import deep_update, json_load, validate
 from qmk.keyboard import config_h, rules_mk
-from qmk.keymap import list_keymaps, locate_keymap
 from qmk.commands import parse_configurator_json
 from qmk.makefile import parse_rules_mk_file
 from qmk.math import compute
@@ -99,10 +98,6 @@ def info_json(keyboard):
         'maintainer': 'qmk',
     }
 
-    # Populate the list of JSON keymaps
-    for keymap in list_keymaps(keyboard, c=False, fullpath=True):
-        info_data['keymaps'][keymap.name] = {'url': f'https://raw.githubusercontent.com/qmk/qmk_firmware/master/{keymap}/keymap.json'}
-
     # Populate layout data
     layouts, aliases = _search_keyboard_h(keyboard)
 
@@ -872,6 +867,9 @@ def find_info_json(keyboard):
 def keymap_json_config(keyboard, keymap):
     """Extract keymap level config
     """
+    # TODO: resolve keymap.py and info.py circular dependencies
+    from qmk.keymap import locate_keymap
+
     keymap_folder = locate_keymap(keyboard, keymap).parent
 
     km_info_json = parse_configurator_json(keymap_folder / 'keymap.json')
@@ -881,6 +879,9 @@ def keymap_json_config(keyboard, keymap):
 def keymap_json(keyboard, keymap):
     """Generate the info.json data for a specific keymap.
     """
+    # TODO: resolve keymap.py and info.py circular dependencies
+    from qmk.keymap import locate_keymap
+
     keymap_folder = locate_keymap(keyboard, keymap).parent
 
     # Files to scan
diff --git a/lib/python/qmk/keymap.py b/lib/python/qmk/keymap.py
index 315af35b73..ce518e379d 100644
--- a/lib/python/qmk/keymap.py
+++ b/lib/python/qmk/keymap.py
@@ -12,8 +12,9 @@ from pygments.token import Token
 from pygments import lex
 
 import qmk.path
-from qmk.keyboard import find_keyboard_from_dir, rules_mk, keyboard_folder
+from qmk.keyboard import find_keyboard_from_dir, keyboard_folder
 from qmk.errors import CppError
+from qmk.info import info_json
 
 # The `keymap.c` template to use when a keyboard doesn't have its own
 DEFAULT_KEYMAP_C = """#include QMK_KEYBOARD_H
@@ -374,11 +375,11 @@ def locate_keymap(keyboard, keymap):
         return keymap_path
 
     # Check community layouts as a fallback
-    rules = rules_mk(keyboard)
+    info = info_json(keyboard)
 
-    if "LAYOUTS" in rules:
-        for layout in rules["LAYOUTS"].split():
-            community_layout = Path('layouts/community') / layout / keymap
+    for community_parent in Path('layouts').glob('*/'):
+        for layout in info.get("community_layouts", []):
+            community_layout = community_parent / layout / keymap
             if community_layout.exists():
                 if (community_layout / 'keymap.json').exists():
                     return community_layout / 'keymap.json'
@@ -408,37 +409,36 @@ def list_keymaps(keyboard, c=True, json=True, additional_files=None, fullpath=Fa
     Returns:
         a sorted list of valid keymap names.
     """
-    # parse all the rules.mk files for the keyboard
-    rules = rules_mk(keyboard)
     names = set()
 
-    if rules is not None:
-        keyboards_dir = Path('keyboards')
-        kb_path = keyboards_dir / keyboard
+    keyboards_dir = Path('keyboards')
+    kb_path = keyboards_dir / keyboard
 
-        # walk up the directory tree until keyboards_dir
-        # and collect all directories' name with keymap.c file in it
-        while kb_path != keyboards_dir:
-            keymaps_dir = kb_path / "keymaps"
+    # walk up the directory tree until keyboards_dir
+    # and collect all directories' name with keymap.c file in it
+    while kb_path != keyboards_dir:
+        keymaps_dir = kb_path / "keymaps"
 
-            if keymaps_dir.is_dir():
-                for keymap in keymaps_dir.iterdir():
+        if keymaps_dir.is_dir():
+            for keymap in keymaps_dir.iterdir():
+                if is_keymap_dir(keymap, c, json, additional_files):
+                    keymap = keymap if fullpath else keymap.name
+                    names.add(keymap)
+
+        kb_path = kb_path.parent
+
+    # Check community layouts as a fallback
+    info = info_json(keyboard)
+
+    for community_parent in Path('layouts').glob('*/'):
+        for layout in info.get("community_layouts", []):
+            cl_path = community_parent / layout
+            if cl_path.is_dir():
+                for keymap in cl_path.iterdir():
                     if is_keymap_dir(keymap, c, json, additional_files):
                         keymap = keymap if fullpath else keymap.name
                         names.add(keymap)
 
-            kb_path = kb_path.parent
-
-        # if community layouts are supported, get them
-        if "LAYOUTS" in rules:
-            for layout in rules["LAYOUTS"].split():
-                cl_path = Path('layouts/community') / layout
-                if cl_path.is_dir():
-                    for keymap in cl_path.iterdir():
-                        if is_keymap_dir(keymap, c, json, additional_files):
-                            keymap = keymap if fullpath else keymap.name
-                            names.add(keymap)
-
     return sorted(names)
 
 
-- 
cgit v1.2.3


From e11235ee14f9cd3fc45b836eec99ed312cb137dd Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Wed, 11 Jan 2023 02:13:32 +0000
Subject: De-duplicate platform detection (#19545)

---
 lib/python/qmk/info.py | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'lib/python/qmk')

diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py
index af3b0f3091..789d05850c 100644
--- a/lib/python/qmk/info.py
+++ b/lib/python/qmk/info.py
@@ -751,6 +751,7 @@ def arm_processor_rules(info_data, rules):
     """
     info_data['processor_type'] = 'arm'
     info_data['protocol'] = 'ChibiOS'
+    info_data['platform_key'] = 'chibios'
 
     if 'STM32' in info_data['processor']:
         info_data['platform'] = 'STM32'
@@ -758,6 +759,7 @@ def arm_processor_rules(info_data, rules):
         info_data['platform'] = rules['MCU_SERIES']
     elif 'ARM_ATSAM' in rules:
         info_data['platform'] = 'ARM_ATSAM'
+        info_data['platform_key'] = 'arm_atsam'
 
     return info_data
 
@@ -767,6 +769,7 @@ def avr_processor_rules(info_data, rules):
     """
     info_data['processor_type'] = 'avr'
     info_data['platform'] = rules['ARCH'] if 'ARCH' in rules else 'unknown'
+    info_data['platform_key'] = 'avr'
     info_data['protocol'] = 'V-USB' if info_data['processor'] in VUSB_PROCESSORS else 'LUFA'
 
     # FIXME(fauxpark/anyone): Eventually we should detect the protocol by looking at PROTOCOL inherited from mcu_selection.mk:
-- 
cgit v1.2.3


From 46c85c93f05003ecc9d5b9266bc78e98cc7a843b Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Wed, 11 Jan 2023 19:58:27 +0000
Subject: Revert "De-duplicate platform detection (#19545)" (#19564)

This reverts commit e11235ee14f9cd3fc45b836eec99ed312cb137dd.
---
 lib/python/qmk/info.py | 3 ---
 1 file changed, 3 deletions(-)

(limited to 'lib/python/qmk')

diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py
index 789d05850c..af3b0f3091 100644
--- a/lib/python/qmk/info.py
+++ b/lib/python/qmk/info.py
@@ -751,7 +751,6 @@ def arm_processor_rules(info_data, rules):
     """
     info_data['processor_type'] = 'arm'
     info_data['protocol'] = 'ChibiOS'
-    info_data['platform_key'] = 'chibios'
 
     if 'STM32' in info_data['processor']:
         info_data['platform'] = 'STM32'
@@ -759,7 +758,6 @@ def arm_processor_rules(info_data, rules):
         info_data['platform'] = rules['MCU_SERIES']
     elif 'ARM_ATSAM' in rules:
         info_data['platform'] = 'ARM_ATSAM'
-        info_data['platform_key'] = 'arm_atsam'
 
     return info_data
 
@@ -769,7 +767,6 @@ def avr_processor_rules(info_data, rules):
     """
     info_data['processor_type'] = 'avr'
     info_data['platform'] = rules['ARCH'] if 'ARCH' in rules else 'unknown'
-    info_data['platform_key'] = 'avr'
     info_data['protocol'] = 'V-USB' if info_data['processor'] in VUSB_PROCESSORS else 'LUFA'
 
     # FIXME(fauxpark/anyone): Eventually we should detect the protocol by looking at PROTOCOL inherited from mcu_selection.mk:
-- 
cgit v1.2.3


From 45851a10f66119ceff3baadde27f68e287eff481 Mon Sep 17 00:00:00 2001
From: David Hoelscher <infinityis@users.noreply.github.com>
Date: Sat, 14 Jan 2023 04:24:54 -0600
Subject: Add RGB565 and RGB888 color support to Quantum Painter (#19382)

---
 lib/python/qmk/painter.py | 80 ++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 75 insertions(+), 5 deletions(-)

(limited to 'lib/python/qmk')

diff --git a/lib/python/qmk/painter.py b/lib/python/qmk/painter.py
index d0cc1dddec..7ecdc55404 100644
--- a/lib/python/qmk/painter.py
+++ b/lib/python/qmk/painter.py
@@ -7,6 +7,20 @@ from PIL import Image, ImageOps
 
 # The list of valid formats Quantum Painter supports
 valid_formats = {
+    'rgb888': {
+        'image_format': 'IMAGE_FORMAT_RGB888',
+        'bpp': 24,
+        'has_palette': False,
+        'num_colors': 16777216,
+        'image_format_byte': 0x09,  # see qp_internal_formats.h
+    },
+    'rgb565': {
+        'image_format': 'IMAGE_FORMAT_RGB565',
+        'bpp': 16,
+        'has_palette': False,
+        'num_colors': 65536,
+        'image_format_byte': 0x08,  # see qp_internal_formats.h
+    },
     'pal256': {
         'image_format': 'IMAGE_FORMAT_PALETTE',
         'bpp': 8,
@@ -144,19 +158,33 @@ def convert_requested_format(im, format):
     ncolors = format["num_colors"]
     image_format = format["image_format"]
 
-    # Ensure we have a valid number of colors for the palette
-    if ncolors <= 0 or ncolors > 256 or (ncolors & (ncolors - 1) != 0):
-        raise ValueError("Number of colors must be 2, 4, 16, or 256.")
-
     # Work out where we're getting the bytes from
     if image_format == 'IMAGE_FORMAT_GRAYSCALE':
+        # Ensure we have a valid number of colors for the palette
+        if ncolors <= 0 or ncolors > 256 or (ncolors & (ncolors - 1) != 0):
+            raise ValueError("Number of colors must be 2, 4, 16, or 256.")
         # If mono, convert input to grayscale, then to RGB, then grab the raw bytes corresponding to the intensity of the red channel
         im = ImageOps.grayscale(im)
         im = im.convert("RGB")
     elif image_format == 'IMAGE_FORMAT_PALETTE':
+        # Ensure we have a valid number of colors for the palette
+        if ncolors <= 0 or ncolors > 256 or (ncolors & (ncolors - 1) != 0):
+            raise ValueError("Number of colors must be 2, 4, 16, or 256.")
         # If color, convert input to RGB, palettize based on the supplied number of colors, then get the raw palette bytes
         im = im.convert("RGB")
         im = im.convert("P", palette=Image.ADAPTIVE, colors=ncolors)
+    elif image_format == 'IMAGE_FORMAT_RGB565':
+        # Ensure we have a valid number of colors for the palette
+        if ncolors != 65536:
+            raise ValueError("Number of colors must be 65536.")
+        # If color, convert input to RGB
+        im = im.convert("RGB")
+    elif image_format == 'IMAGE_FORMAT_RGB888':
+        # Ensure we have a valid number of colors for the palette
+        if ncolors != 1677216:
+            raise ValueError("Number of colors must be 16777216.")
+        # If color, convert input to RGB
+        im = im.convert("RGB")
 
     return im
 
@@ -170,8 +198,12 @@ def convert_image_bytes(im, format):
     image_format = format["image_format"]
     shifter = int(math.log2(ncolors))
     pixels_per_byte = int(8 / math.log2(ncolors))
+    bytes_per_pixel = math.ceil(math.log2(ncolors) / 8)
     (width, height) = im.size
-    expected_byte_count = ((width * height) + (pixels_per_byte - 1)) // pixels_per_byte
+    if (pixels_per_byte != 0):
+        expected_byte_count = ((width * height) + (pixels_per_byte - 1)) // pixels_per_byte
+    else:
+        expected_byte_count = width * height * bytes_per_pixel
 
     if image_format == 'IMAGE_FORMAT_GRAYSCALE':
         # Take the red channel
@@ -212,6 +244,44 @@ def convert_image_bytes(im, format):
                     byte = byte | ((image_bytes[byte_offset] & (ncolors - 1)) << int(n * shifter))
             bytearray.append(byte)
 
+    if image_format == 'IMAGE_FORMAT_RGB565':
+        # Take the red, green, and blue channels
+        image_bytes_red = im.tobytes("raw", "R")
+        image_bytes_green = im.tobytes("raw", "G")
+        image_bytes_blue = im.tobytes("raw", "B")
+        image_pixels_len = len(image_bytes_red)
+
+        # No palette
+        palette = None
+
+        bytearray = []
+        for x in range(image_pixels_len):
+            # 5 bits of red, 3 MSb of green
+            byte = ((image_bytes_red[x] >> 3 & 0x1F) << 3) + (image_bytes_green[x] >> 5 & 0x07)
+            bytearray.append(byte)
+            # 3 LSb of green, 5 bits of blue
+            byte = ((image_bytes_green[x] >> 2 & 0x07) << 5) + (image_bytes_blue[x] >> 3 & 0x1F)
+            bytearray.append(byte)
+
+    if image_format == 'IMAGE_FORMAT_RGB888':
+        # Take the red, green, and blue channels
+        image_bytes_red = im.tobytes("raw", "R")
+        image_bytes_green = im.tobytes("raw", "G")
+        image_bytes_blue = im.tobytes("raw", "B")
+        image_pixels_len = len(image_bytes_red)
+
+        # No palette
+        palette = None
+
+        bytearray = []
+        for x in range(image_pixels_len):
+            byte = image_bytes_red[x]
+            bytearray.append(byte)
+            byte = image_bytes_green[x]
+            bytearray.append(byte)
+            byte = image_bytes_blue[x]
+            bytearray.append(byte)
+
     if len(bytearray) != expected_byte_count:
         raise Exception(f"Wrong byte count, was {len(bytearray)}, expected {expected_byte_count}")
 
-- 
cgit v1.2.3


From 88ec588ae7f6d36b23578135cde8b7a5160ff8b7 Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Wed, 18 Jan 2023 23:44:41 +0000
Subject: Remove `make all-<platform>` build targets (#19496)

---
 lib/python/qmk/cli/mass_compile.py | 2 +-
 lib/python/qmk/cli/multibuild.py   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib/python/qmk')

diff --git a/lib/python/qmk/cli/mass_compile.py b/lib/python/qmk/cli/mass_compile.py
index a98f7ad482..d8dda0cac3 100755
--- a/lib/python/qmk/cli/mass_compile.py
+++ b/lib/python/qmk/cli/mass_compile.py
@@ -134,7 +134,7 @@ all: {keyboard_safe}_{keymap_name}_binary
 {keyboard_safe}_{keymap_name}_binary:
 	@rm -f "{QMK_FIRMWARE}/.build/failed.log.{keyboard_safe}.{keymap_name}" || true
 	@echo "Compiling QMK Firmware for target: '{keyboard_name}:{keymap_name}'..." >>"{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}"
-	+@$(MAKE) -C "{QMK_FIRMWARE}" -f "{QMK_FIRMWARE}/builddefs/build_keyboard.mk" KEYBOARD="{keyboard_name}" KEYMAP="{keymap_name}" REQUIRE_PLATFORM_KEY= COLOR=true SILENT=false {' '.join(cli.args.env)} \\
+	+@$(MAKE) -C "{QMK_FIRMWARE}" -f "{QMK_FIRMWARE}/builddefs/build_keyboard.mk" KEYBOARD="{keyboard_name}" KEYMAP="{keymap_name}" COLOR=true SILENT=false {' '.join(cli.args.env)} \\
 		>>"{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}.{keymap_name}" 2>&1 \\
 		|| cp "{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}.{keymap_name}" "{QMK_FIRMWARE}/.build/failed.log.{os.getpid()}.{keyboard_safe}.{keymap_name}"
 	@{{ grep '\[ERRORS\]' "{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}.{keymap_name}" >/dev/null 2>&1 && printf "Build %-64s \e[1;31m[ERRORS]\e[0m\\n" "{keyboard_name}:{keymap_name}" ; }} \\
diff --git a/lib/python/qmk/cli/multibuild.py b/lib/python/qmk/cli/multibuild.py
index 5e0f0b5188..22b68f43c8 100755
--- a/lib/python/qmk/cli/multibuild.py
+++ b/lib/python/qmk/cli/multibuild.py
@@ -71,7 +71,7 @@ all: {keyboard_safe}_binary
 {keyboard_safe}_binary:
 	@rm -f "{QMK_FIRMWARE}/.build/failed.log.{keyboard_safe}" || true
 	@echo "Compiling QMK Firmware for target: '{keyboard_name}:{cli.args.keymap}'..." >>"{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}"
-	+@$(MAKE) -C "{QMK_FIRMWARE}" -f "{QMK_FIRMWARE}/builddefs/build_keyboard.mk" KEYBOARD="{keyboard_name}" KEYMAP="{cli.args.keymap}" REQUIRE_PLATFORM_KEY= COLOR=true SILENT=false {' '.join(cli.args.env)} \\
+	+@$(MAKE) -C "{QMK_FIRMWARE}" -f "{QMK_FIRMWARE}/builddefs/build_keyboard.mk" KEYBOARD="{keyboard_name}" KEYMAP="{cli.args.keymap}" COLOR=true SILENT=false {' '.join(cli.args.env)} \\
 		>>"{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}" 2>&1 \\
 		|| cp "{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}" "{QMK_FIRMWARE}/.build/failed.log.{os.getpid()}.{keyboard_safe}"
 	@{{ grep '\[ERRORS\]' "{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}" >/dev/null 2>&1 && printf "Build %-64s \e[1;31m[ERRORS]\e[0m\\n" "{keyboard_name}:{cli.args.keymap}" ; }} \\
-- 
cgit v1.2.3


From 4723f308adb7d1d4a4136722e2f576a398695559 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Thu, 19 Jan 2023 10:56:15 +1100
Subject: Remove CLI commands: `multibuild`, `cformat`, `fileformat`,
 `pyformat`. (#19629)

---
 lib/python/qmk/cli/__init__.py   |   4 --
 lib/python/qmk/cli/cformat.py    |  28 -----------
 lib/python/qmk/cli/fileformat.py |  23 ---------
 lib/python/qmk/cli/multibuild.py | 106 ---------------------------------------
 lib/python/qmk/cli/pyformat.py   |  24 ---------
 5 files changed, 185 deletions(-)
 delete mode 100755 lib/python/qmk/cli/cformat.py
 delete mode 100755 lib/python/qmk/cli/fileformat.py
 delete mode 100755 lib/python/qmk/cli/multibuild.py
 delete mode 100755 lib/python/qmk/cli/pyformat.py

(limited to 'lib/python/qmk')

diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py
index 16edf88ad6..778eccada8 100644
--- a/lib/python/qmk/cli/__init__.py
+++ b/lib/python/qmk/cli/__init__.py
@@ -34,13 +34,11 @@ subcommands = [
     'qmk.cli.bux',
     'qmk.cli.c2json',
     'qmk.cli.cd',
-    'qmk.cli.cformat',
     'qmk.cli.chibios.confmigrate',
     'qmk.cli.clean',
     'qmk.cli.compile',
     'qmk.cli.docs',
     'qmk.cli.doctor',
-    'qmk.cli.fileformat',
     'qmk.cli.flash',
     'qmk.cli.format.c',
     'qmk.cli.format.json',
@@ -75,11 +73,9 @@ subcommands = [
     'qmk.cli.list.layouts',
     'qmk.cli.mass_compile',
     'qmk.cli.migrate',
-    'qmk.cli.multibuild',
     'qmk.cli.new.keyboard',
     'qmk.cli.new.keymap',
     'qmk.cli.painter',
-    'qmk.cli.pyformat',
     'qmk.cli.pytest',
     'qmk.cli.via2json',
 ]
diff --git a/lib/python/qmk/cli/cformat.py b/lib/python/qmk/cli/cformat.py
deleted file mode 100755
index 9d0ecaeba3..0000000000
--- a/lib/python/qmk/cli/cformat.py
+++ /dev/null
@@ -1,28 +0,0 @@
-"""Point people to the new command name.
-"""
-import sys
-from pathlib import Path
-
-from milc import cli
-
-
-@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Flag only, don't automatically format.")
-@cli.argument('-b', '--base-branch', default='origin/master', help='Branch to compare to diffs to.')
-@cli.argument('-a', '--all-files', arg_only=True, action='store_true', help='Format all core files.')
-@cli.argument('--core-only', arg_only=True, action='store_true', help='Format core files only.')
-@cli.argument('files', nargs='*', arg_only=True, help='Filename(s) to format.')
-@cli.subcommand('Pointer to the new command name: qmk format-c.', hidden=True)
-def cformat(cli):
-    """Pointer to the new command name: qmk format-c.
-    """
-    cli.log.warning('"qmk cformat" has been renamed to "qmk format-c". Please use the new command in the future.')
-    argv = [sys.executable, *sys.argv]
-    argv[argv.index('cformat')] = 'format-c'
-    script_path = Path(argv[1])
-    script_path_exe = Path(f'{argv[1]}.exe')
-
-    if not script_path.exists() and script_path_exe.exists():
-        # For reasons I don't understand ".exe" is stripped from the script name on windows.
-        argv[1] = str(script_path_exe)
-
-    return cli.run(argv, capture_output=False).returncode
diff --git a/lib/python/qmk/cli/fileformat.py b/lib/python/qmk/cli/fileformat.py
deleted file mode 100755
index cee4ba1acd..0000000000
--- a/lib/python/qmk/cli/fileformat.py
+++ /dev/null
@@ -1,23 +0,0 @@
-"""Point people to the new command name.
-"""
-import sys
-from pathlib import Path
-
-from milc import cli
-
-
-@cli.subcommand('Pointer to the new command name: qmk format-text.', hidden=True)
-def fileformat(cli):
-    """Pointer to the new command name: qmk format-text.
-    """
-    cli.log.warning('"qmk fileformat" has been renamed to "qmk format-text". Please use the new command in the future.')
-    argv = [sys.executable, *sys.argv]
-    argv[argv.index('fileformat')] = 'format-text'
-    script_path = Path(argv[1])
-    script_path_exe = Path(f'{argv[1]}.exe')
-
-    if not script_path.exists() and script_path_exe.exists():
-        # For reasons I don't understand ".exe" is stripped from the script name on windows.
-        argv[1] = str(script_path_exe)
-
-    return cli.run(argv, capture_output=False).returncode
diff --git a/lib/python/qmk/cli/multibuild.py b/lib/python/qmk/cli/multibuild.py
deleted file mode 100755
index 22b68f43c8..0000000000
--- a/lib/python/qmk/cli/multibuild.py
+++ /dev/null
@@ -1,106 +0,0 @@
-"""Compile all keyboards.
-
-This will compile everything in parallel, for testing purposes.
-"""
-import os
-import re
-from pathlib import Path
-from subprocess import DEVNULL
-
-from milc import cli
-
-from qmk.constants import QMK_FIRMWARE
-from qmk.commands import _find_make, get_make_parallel_args
-import qmk.keyboard
-import qmk.keymap
-
-
-def _make_rules_mk_filter(key, value):
-    def _rules_mk_filter(keyboard_name):
-        rules_mk = qmk.keyboard.rules_mk(keyboard_name)
-        return True if key in rules_mk and rules_mk[key].lower() == str(value).lower() else False
-
-    return _rules_mk_filter
-
-
-def _is_split(keyboard_name):
-    rules_mk = qmk.keyboard.rules_mk(keyboard_name)
-    return True if 'SPLIT_KEYBOARD' in rules_mk and rules_mk['SPLIT_KEYBOARD'].lower() == 'yes' else False
-
-
-@cli.argument('-t', '--no-temp', arg_only=True, action='store_true', help="Remove temporary files during build.")
-@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")
-@cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.")
-@cli.argument('-f', '--filter', arg_only=True, action='append', default=[], help="Filter the list of keyboards based on the supplied value in rules.mk. Supported format is 'SPLIT_KEYBOARD=yes'. May be passed multiple times.")
-@cli.argument('-km', '--keymap', type=str, default='default', help="The keymap name to build. Default is 'default'.")
-@cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")
-@cli.subcommand('Compile QMK Firmware for all keyboards.', hidden=False if cli.config.user.developer else True)
-def multibuild(cli):
-    """Compile QMK Firmware against all keyboards.
-    """
-
-    make_cmd = _find_make()
-    if cli.args.clean:
-        cli.run([make_cmd, 'clean'], capture_output=False, stdin=DEVNULL)
-
-    builddir = Path(QMK_FIRMWARE) / '.build'
-    makefile = builddir / 'parallel_kb_builds.mk'
-
-    keyboard_list = qmk.keyboard.list_keyboards()
-
-    filter_re = re.compile(r'^(?P<key>[A-Z0-9_]+)\s*=\s*(?P<value>[^#]+)$')
-    for filter_txt in cli.args.filter:
-        f = filter_re.match(filter_txt)
-        if f is not None:
-            keyboard_list = filter(_make_rules_mk_filter(f.group('key'), f.group('value')), keyboard_list)
-
-    keyboard_list = list(sorted(keyboard_list))
-
-    if len(keyboard_list) == 0:
-        return
-
-    builddir.mkdir(parents=True, exist_ok=True)
-    with open(makefile, "w") as f:
-        for keyboard_name in keyboard_list:
-            if qmk.keymap.locate_keymap(keyboard_name, cli.args.keymap) is not None:
-                keyboard_safe = keyboard_name.replace('/', '_')
-                # yapf: disable
-                f.write(
-                    f"""\
-all: {keyboard_safe}_binary
-{keyboard_safe}_binary:
-	@rm -f "{QMK_FIRMWARE}/.build/failed.log.{keyboard_safe}" || true
-	@echo "Compiling QMK Firmware for target: '{keyboard_name}:{cli.args.keymap}'..." >>"{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}"
-	+@$(MAKE) -C "{QMK_FIRMWARE}" -f "{QMK_FIRMWARE}/builddefs/build_keyboard.mk" KEYBOARD="{keyboard_name}" KEYMAP="{cli.args.keymap}" COLOR=true SILENT=false {' '.join(cli.args.env)} \\
-		>>"{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}" 2>&1 \\
-		|| cp "{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}" "{QMK_FIRMWARE}/.build/failed.log.{os.getpid()}.{keyboard_safe}"
-	@{{ grep '\[ERRORS\]' "{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}" >/dev/null 2>&1 && printf "Build %-64s \e[1;31m[ERRORS]\e[0m\\n" "{keyboard_name}:{cli.args.keymap}" ; }} \\
-		|| {{ grep '\[WARNINGS\]' "{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}" >/dev/null 2>&1 && printf "Build %-64s \e[1;33m[WARNINGS]\e[0m\\n" "{keyboard_name}:{cli.args.keymap}" ; }} \\
-		|| printf "Build %-64s \e[1;32m[OK]\e[0m\\n" "{keyboard_name}:{cli.args.keymap}"
-	@rm -f "{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}" || true
-"""# noqa
-                )
-                # yapf: enable
-
-                if cli.args.no_temp:
-                    # yapf: disable
-                    f.write(
-                        f"""\
-	@rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{cli.args.keymap}.elf" 2>/dev/null || true
-	@rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{cli.args.keymap}.map" 2>/dev/null || true
-	@rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{cli.args.keymap}.hex" 2>/dev/null || true
-	@rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{cli.args.keymap}.bin" 2>/dev/null || true
-	@rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{cli.args.keymap}.uf2" 2>/dev/null || true
-	@rm -rf "{QMK_FIRMWARE}/.build/obj_{keyboard_safe}" || true
-	@rm -rf "{QMK_FIRMWARE}/.build/obj_{keyboard_safe}_{cli.args.keymap}" || true
-"""# noqa
-                    )
-                    # yapf: enable
-                f.write('\n')
-
-    cli.run([make_cmd, *get_make_parallel_args(cli.args.parallel), '-f', makefile.as_posix(), 'all'], capture_output=False, stdin=DEVNULL)
-
-    # Check for failures
-    failures = [f for f in builddir.glob(f'failed.log.{os.getpid()}.*')]
-    if len(failures) > 0:
-        return False
diff --git a/lib/python/qmk/cli/pyformat.py b/lib/python/qmk/cli/pyformat.py
deleted file mode 100755
index c624f74aeb..0000000000
--- a/lib/python/qmk/cli/pyformat.py
+++ /dev/null
@@ -1,24 +0,0 @@
-"""Point people to the new command name.
-"""
-import sys
-from pathlib import Path
-
-from milc import cli
-
-
-@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually format.")
-@cli.subcommand('Pointer to the new command name: qmk format-python.', hidden=False if cli.config.user.developer else True)
-def pyformat(cli):
-    """Pointer to the new command name: qmk format-python.
-    """
-    cli.log.warning('"qmk pyformat" has been renamed to "qmk format-python". Please use the new command in the future.')
-    argv = [sys.executable, *sys.argv]
-    argv[argv.index('pyformat')] = 'format-python'
-    script_path = Path(argv[1])
-    script_path_exe = Path(f'{argv[1]}.exe')
-
-    if not script_path.exists() and script_path_exe.exists():
-        # For reasons I don't understand ".exe" is stripped from the script name on windows.
-        argv[1] = str(script_path_exe)
-
-    return cli.run(argv, capture_output=False).returncode
-- 
cgit v1.2.3


From a1f253cbef309906c989cfb3e123ee8fb3172bb4 Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Thu, 19 Jan 2023 00:24:13 +0000
Subject: `qmk compile`/`qmk flash` - Validate keymap argument (#19530)

---
 lib/python/qmk/cli/compile.py | 17 ++++++++++++++++-
 lib/python/qmk/cli/flash.py   | 17 ++++++++++++++++-
 2 files changed, 32 insertions(+), 2 deletions(-)

(limited to 'lib/python/qmk')

diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py
index 9e7629906f..f43e5f32de 100755
--- a/lib/python/qmk/cli/compile.py
+++ b/lib/python/qmk/cli/compile.py
@@ -10,7 +10,17 @@ import qmk.path
 from qmk.decorators import automagic_keyboard, automagic_keymap
 from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json, build_environment
 from qmk.keyboard import keyboard_completer, keyboard_folder
-from qmk.keymap import keymap_completer
+from qmk.keymap import keymap_completer, locate_keymap
+
+
+def _is_keymap_target(keyboard, keymap):
+    if keymap == 'all':
+        return True
+
+    if locate_keymap(keyboard, keymap):
+        return True
+
+    return False
 
 
 @cli.argument('filename', nargs='?', arg_only=True, type=qmk.path.FileType('r'), completer=FilesCompleter('.json'), help='The configurator export to compile')
@@ -43,6 +53,11 @@ def compile(cli):
 
     elif cli.config.compile.keyboard and cli.config.compile.keymap:
         # Generate the make command for a specific keyboard/keymap.
+        if not _is_keymap_target(cli.config.compile.keyboard, cli.config.compile.keymap):
+            cli.log.error('Invalid keymap argument.')
+            cli.print_help()
+            return False
+
         if cli.args.clean:
             commands.append(create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, 'clean', **envs))
         commands.append(create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, parallel=cli.config.compile.parallel, **envs))
diff --git a/lib/python/qmk/cli/flash.py b/lib/python/qmk/cli/flash.py
index 52defb5f0d..8724f26889 100644
--- a/lib/python/qmk/cli/flash.py
+++ b/lib/python/qmk/cli/flash.py
@@ -11,10 +11,20 @@ import qmk.path
 from qmk.decorators import automagic_keyboard, automagic_keymap
 from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json, build_environment
 from qmk.keyboard import keyboard_completer, keyboard_folder
-from qmk.keymap import keymap_completer
+from qmk.keymap import keymap_completer, locate_keymap
 from qmk.flashers import flasher
 
 
+def _is_keymap_target(keyboard, keymap):
+    if keymap == 'all':
+        return True
+
+    if locate_keymap(keyboard, keymap):
+        return True
+
+    return False
+
+
 def _list_bootloaders():
     """Prints the available bootloaders listed in docs.qmk.fm.
     """
@@ -98,6 +108,11 @@ def flash(cli):
 
     elif cli.config.flash.keyboard and cli.config.flash.keymap:
         # Generate the make command for a specific keyboard/keymap.
+        if not _is_keymap_target(cli.config.flash.keyboard, cli.config.flash.keymap):
+            cli.log.error('Invalid keymap argument.')
+            cli.print_help()
+            return False
+
         if cli.args.clean:
             commands.append(create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, 'clean', **envs))
         commands.append(create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, **envs))
-- 
cgit v1.2.3


From 0ce3f6bcfe8241e51dd6936d24e6a88c907c535d Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Thu, 19 Jan 2023 00:27:00 +0000
Subject: De-duplicate platform detection (#19603)

---
 lib/python/qmk/info.py | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'lib/python/qmk')

diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py
index af3b0f3091..789d05850c 100644
--- a/lib/python/qmk/info.py
+++ b/lib/python/qmk/info.py
@@ -751,6 +751,7 @@ def arm_processor_rules(info_data, rules):
     """
     info_data['processor_type'] = 'arm'
     info_data['protocol'] = 'ChibiOS'
+    info_data['platform_key'] = 'chibios'
 
     if 'STM32' in info_data['processor']:
         info_data['platform'] = 'STM32'
@@ -758,6 +759,7 @@ def arm_processor_rules(info_data, rules):
         info_data['platform'] = rules['MCU_SERIES']
     elif 'ARM_ATSAM' in rules:
         info_data['platform'] = 'ARM_ATSAM'
+        info_data['platform_key'] = 'arm_atsam'
 
     return info_data
 
@@ -767,6 +769,7 @@ def avr_processor_rules(info_data, rules):
     """
     info_data['processor_type'] = 'avr'
     info_data['platform'] = rules['ARCH'] if 'ARCH' in rules else 'unknown'
+    info_data['platform_key'] = 'avr'
     info_data['protocol'] = 'V-USB' if info_data['processor'] in VUSB_PROCESSORS else 'LUFA'
 
     # FIXME(fauxpark/anyone): Eventually we should detect the protocol by looking at PROTOCOL inherited from mcu_selection.mk:
-- 
cgit v1.2.3


From 0b25528b6bc433e8609d4e12fb108b4ee7865197 Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Thu, 19 Jan 2023 00:27:11 +0000
Subject: Fix 'No LAYOUTs defined' check (#19537)

---
 lib/python/qmk/info.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

(limited to 'lib/python/qmk')

diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py
index 789d05850c..e73d825696 100644
--- a/lib/python/qmk/info.py
+++ b/lib/python/qmk/info.py
@@ -52,7 +52,7 @@ def _validate(keyboard, info_data):
     community_layouts_names = list(map(lambda layout: f'LAYOUT_{layout}', community_layouts))
 
     # Make sure we have at least one layout
-    if len(layouts) == 0:
+    if len(layouts) == 0 or all(not layout.get('json_layout', False) for layout in layouts.values()):
         _log_error(info_data, 'No LAYOUTs defined! Need at least one layout defined in info.json.')
 
     # Providing only LAYOUT_all "because I define my layouts in a 3rd party tool"
@@ -107,6 +107,7 @@ def info_json(keyboard):
     for layout_name, layout_json in layouts.items():
         if not layout_name.startswith('LAYOUT_kc'):
             layout_json['c_macro'] = True
+            layout_json['json_layout'] = False
             info_data['layouts'][layout_name] = layout_json
 
     # Merge in the data from info.json, config.h, and rules.mk
@@ -824,6 +825,7 @@ def merge_info_jsons(keyboard, info_data):
                     msg = 'Number of keys for %s does not match! info.json specifies %d keys, C macro specifies %d'
                     _log_error(info_data, msg % (layout_name, len(layout['layout']), len(info_data['layouts'][layout_name]['layout'])))
                 else:
+                    info_data['layouts'][layout_name]['json_layout'] = True
                     for new_key, existing_key in zip(layout['layout'], info_data['layouts'][layout_name]['layout']):
                         existing_key.update(new_key)
             else:
@@ -831,6 +833,7 @@ def merge_info_jsons(keyboard, info_data):
                     _log_error(info_data, f'Layout "{layout_name}" has no "matrix" definition in either "info.json" or "<keyboard>.h"!')
                 else:
                     layout['c_macro'] = False
+                    layout['json_layout'] = True
                     info_data['layouts'][layout_name] = layout
 
         # Update info_data with the new data
-- 
cgit v1.2.3


From 4973950ddcef28d94a1fc589951e024a91a240d7 Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Thu, 19 Jan 2023 10:25:47 +0000
Subject: Print distro in doctor output (#19633)

---
 lib/python/qmk/cli/doctor/check.py   | 18 ++++++++++++++++++
 lib/python/qmk/cli/doctor/linux.py   | 23 ++++++++++++++++-------
 lib/python/qmk/cli/doctor/windows.py |  8 +++++++-
 3 files changed, 41 insertions(+), 8 deletions(-)

(limited to 'lib/python/qmk')

diff --git a/lib/python/qmk/cli/doctor/check.py b/lib/python/qmk/cli/doctor/check.py
index 426876e98a..cd69cdd11c 100644
--- a/lib/python/qmk/cli/doctor/check.py
+++ b/lib/python/qmk/cli/doctor/check.py
@@ -158,3 +158,21 @@ def is_executable(command):
 
     cli.log.error("{fg_red}Can't run `%s %s`", command, version_arg)
     return False
+
+
+def release_info(file='/etc/os-release'):
+    """Parse release info to dict
+    """
+    ret = {}
+    try:
+        with open(file) as f:
+            for line in f:
+                if '=' in line:
+                    key, value = map(str.strip, line.split('=', 1))
+                    if value.startswith('"') and value.endswith('"'):
+                        value = value[1:-1]
+                    ret[key] = value
+    except (PermissionError, FileNotFoundError):
+        pass
+
+    return ret
diff --git a/lib/python/qmk/cli/doctor/linux.py b/lib/python/qmk/cli/doctor/linux.py
index 95bafe8c64..f0850d4e64 100644
--- a/lib/python/qmk/cli/doctor/linux.py
+++ b/lib/python/qmk/cli/doctor/linux.py
@@ -7,7 +7,11 @@ from pathlib import Path
 from milc import cli
 
 from qmk.constants import QMK_FIRMWARE, BOOTLOADER_VIDS_PIDS
-from .check import CheckStatus
+from .check import CheckStatus, release_info
+
+
+def _is_wsl():
+    return 'microsoft' in platform.uname().release.lower()
 
 
 def _udev_rule(vid, pid=None, *args):
@@ -130,17 +134,22 @@ def check_modem_manager():
 def os_test_linux():
     """Run the Linux specific tests.
     """
-    # Don't bother with udev on WSL, for now
-    if 'microsoft' in platform.uname().release.lower():
-        cli.log.info("Detected {fg_cyan}Linux (WSL){fg_reset}.")
+    info = release_info()
+    release_id = info.get('PRETTY_NAME', info.get('ID', 'Unknown'))
+    plat = 'WSL, ' if _is_wsl() else ''
 
+    cli.log.info(f"Detected {{fg_cyan}}Linux ({plat}{release_id}){{fg_reset}}.")
+
+    # Don't bother with udev on WSL, for now
+    if _is_wsl():
         # https://github.com/microsoft/WSL/issues/4197
         if QMK_FIRMWARE.as_posix().startswith("/mnt"):
             cli.log.warning("I/O performance on /mnt may be extremely slow.")
             return CheckStatus.WARNING
 
-        return CheckStatus.OK
     else:
-        cli.log.info("Detected {fg_cyan}Linux{fg_reset}.")
+        rc = check_udev_rules()
+        if rc != CheckStatus.OK:
+            return rc
 
-        return check_udev_rules()
+    return CheckStatus.OK
diff --git a/lib/python/qmk/cli/doctor/windows.py b/lib/python/qmk/cli/doctor/windows.py
index 381ab36fde..26bb65374b 100644
--- a/lib/python/qmk/cli/doctor/windows.py
+++ b/lib/python/qmk/cli/doctor/windows.py
@@ -2,7 +2,7 @@ import platform
 
 from milc import cli
 
-from .check import CheckStatus
+from .check import CheckStatus, release_info
 
 
 def os_test_windows():
@@ -11,4 +11,10 @@ def os_test_windows():
     win32_ver = platform.win32_ver()
     cli.log.info("Detected {fg_cyan}Windows %s (%s){fg_reset}.", win32_ver[0], win32_ver[1])
 
+    # MSYS really does not like "/" files - resolve manually
+    file = cli.run(['cygpath', '-m', '/etc/qmk-release']).stdout.strip()
+    qmk_distro_version = release_info(file).get('VERSION', None)
+    if qmk_distro_version:
+        cli.log.info('QMK MSYS version: %s', qmk_distro_version)
+
     return CheckStatus.OK
-- 
cgit v1.2.3


From fe6502f12e12e1fe9691be4b2729cd7ac4c4aac6 Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Fri, 20 Jan 2023 03:38:19 +0000
Subject: Publish keymap.json to API (#19167)

---
 lib/python/qmk/cli/generate/api.py | 49 +++++++++++++++++++++++++-------------
 1 file changed, 32 insertions(+), 17 deletions(-)

(limited to 'lib/python/qmk')

diff --git a/lib/python/qmk/cli/generate/api.py b/lib/python/qmk/cli/generate/api.py
index d662e14e00..11d4616199 100755
--- a/lib/python/qmk/cli/generate/api.py
+++ b/lib/python/qmk/cli/generate/api.py
@@ -43,7 +43,7 @@ def _resolve_keycode_specs(output_folder):
         overall = load_spec(version)
 
         output_file = output_folder / f'constants/keycodes_{version}.json'
-        output_file.write_text(json.dumps(overall, indent=4), encoding='utf-8')
+        output_file.write_text(json.dumps(overall), encoding='utf-8')
 
     for lang in list_languages():
         for version in list_versions(lang):
@@ -64,7 +64,7 @@ def _filtered_copy(src, dst):
         data = json_load(src)
 
         dst = dst.with_suffix('.json')
-        dst.write_text(json.dumps(data, indent=4), encoding='utf-8')
+        dst.write_text(json.dumps(data), encoding='utf-8')
         return dst
 
     return shutil.copy2(src, dst)
@@ -111,29 +111,44 @@ def generate_api(cli):
 
     # Generate and write keyboard specific JSON files
     for keyboard_name in keyboard_list:
-        kb_all[keyboard_name] = info_json(keyboard_name)
-
-        # Populate the list of JSON keymaps
-        for keymap in list_keymaps(keyboard_name, c=False, fullpath=True):
-            kb_all[keyboard_name]['keymaps'][keymap.name] = {'url': f'https://raw.githubusercontent.com/qmk/qmk_firmware/master/{keymap}/keymap.json'}
+        kb_json = info_json(keyboard_name)
+        kb_all[keyboard_name] = kb_json
 
         keyboard_dir = v1_dir / 'keyboards' / keyboard_name
         keyboard_info = keyboard_dir / 'info.json'
         keyboard_readme = keyboard_dir / 'readme.md'
         keyboard_readme_src = find_readme(keyboard_name)
 
+        # Populate the list of JSON keymaps
+        for keymap in list_keymaps(keyboard_name, c=False, fullpath=True):
+            kb_json['keymaps'][keymap.name] = {
+                # TODO: deprecate 'url' as consumer needs to know its potentially hjson
+                'url': f'https://raw.githubusercontent.com/qmk/qmk_firmware/master/{keymap}/keymap.json',
+
+                # Instead consumer should grab from API and not repo directly
+                'path': (keymap / 'keymap.json').as_posix(),
+            }
+
         keyboard_dir.mkdir(parents=True, exist_ok=True)
-        keyboard_json = json.dumps({'last_updated': current_datetime(), 'keyboards': {keyboard_name: kb_all[keyboard_name]}})
+        keyboard_json = json.dumps({'last_updated': current_datetime(), 'keyboards': {keyboard_name: kb_json}})
         if not cli.args.dry_run:
-            keyboard_info.write_text(keyboard_json)
+            keyboard_info.write_text(keyboard_json, encoding='utf-8')
             cli.log.debug('Wrote file %s', keyboard_info)
 
             if keyboard_readme_src:
                 shutil.copyfile(keyboard_readme_src, keyboard_readme)
                 cli.log.debug('Copied %s -> %s', keyboard_readme_src, keyboard_readme)
 
-        if 'usb' in kb_all[keyboard_name]:
-            usb = kb_all[keyboard_name]['usb']
+            # resolve keymaps as json
+            for keymap in kb_json['keymaps']:
+                keymap_hjson = kb_json['keymaps'][keymap]['path']
+                keymap_json = v1_dir / keymap_hjson
+                keymap_json.parent.mkdir(parents=True, exist_ok=True)
+                keymap_json.write_text(json.dumps(json_load(Path(keymap_hjson))), encoding='utf-8')
+                cli.log.debug('Wrote keymap %s', keymap_json)
+
+        if 'usb' in kb_json:
+            usb = kb_json['usb']
 
             if 'vid' in usb and usb['vid'] not in usb_list:
                 usb_list[usb['vid']] = {}
@@ -166,9 +181,9 @@ def generate_api(cli):
     constants_metadata_json = json.dumps({'last_updated': current_datetime(), 'constants': _list_constants(v1_dir)})
 
     if not cli.args.dry_run:
-        keyboard_all_file.write_text(keyboard_all_json)
-        usb_file.write_text(usb_json)
-        keyboard_list_file.write_text(keyboard_list_json)
-        keyboard_aliases_file.write_text(keyboard_aliases_json)
-        keyboard_metadata_file.write_text(keyboard_metadata_json)
-        constants_metadata_file.write_text(constants_metadata_json)
+        keyboard_all_file.write_text(keyboard_all_json, encoding='utf-8')
+        usb_file.write_text(usb_json, encoding='utf-8')
+        keyboard_list_file.write_text(keyboard_list_json, encoding='utf-8')
+        keyboard_aliases_file.write_text(keyboard_aliases_json, encoding='utf-8')
+        keyboard_metadata_file.write_text(keyboard_metadata_json, encoding='utf-8')
+        constants_metadata_file.write_text(constants_metadata_json, encoding='utf-8')
-- 
cgit v1.2.3


From d55b07696b59d6e4bb427e22ed171a4e23c95a2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Pablo=20Mart=C3=ADnez?=
 <58857054+elpekenin@users.noreply.github.com>
Date: Thu, 2 Feb 2023 19:23:27 +0100
Subject: Add commit info to `version.h` (#19542)

* Initial commit

* Fix import order

* Fix deleted code instead of debug print line

* Format

* Update lib/python/qmk/cli/generate/version_h.py

Co-authored-by: Ryan <fauxpark@gmail.com>

* Renaming

* Update lib/python/qmk/cli/generate/version_h.py

Co-authored-by: Joel Challis <git@zvecr.com>

* Update lib/python/qmk/git.py

Co-authored-by: Joel Challis <git@zvecr.com>

---------

Co-authored-by: Ryan <fauxpark@gmail.com>
Co-authored-by: Joel Challis <git@zvecr.com>
---
 lib/python/qmk/cli/generate/version_h.py | 13 ++++++++++---
 lib/python/qmk/git.py                    |  8 ++++++++
 2 files changed, 18 insertions(+), 3 deletions(-)

(limited to 'lib/python/qmk')

diff --git a/lib/python/qmk/cli/generate/version_h.py b/lib/python/qmk/cli/generate/version_h.py
index a75702c529..fd87df3617 100644
--- a/lib/python/qmk/cli/generate/version_h.py
+++ b/lib/python/qmk/cli/generate/version_h.py
@@ -6,7 +6,7 @@ from milc import cli
 
 from qmk.path import normpath
 from qmk.commands import dump_lines
-from qmk.git import git_get_version
+from qmk.git import git_get_qmk_hash, git_get_version, git_is_dirty
 from qmk.constants import GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE
 
 TIME_FMT = '%Y-%m-%d-%H:%M:%S'
@@ -29,23 +29,30 @@ def generate_version_h(cli):
         current_time = strftime(TIME_FMT)
 
     if cli.args.skip_git:
+        git_dirty = False
         git_version = "NA"
+        git_qmk_hash = "NA"
         chibios_version = "NA"
         chibios_contrib_version = "NA"
     else:
+        git_dirty = git_is_dirty()
         git_version = git_get_version() or current_time
+        git_qmk_hash = git_get_qmk_hash() or "Unknown"
         chibios_version = git_get_version("chibios", "os") or current_time
         chibios_contrib_version = git_get_version("chibios-contrib", "os") or current_time
 
     # Build the version.h file.
     version_h_lines = [GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE, '#pragma once']
 
-    version_h_lines.append(f"""
+    version_h_lines.append(
+        f"""
 #define QMK_VERSION "{git_version}"
 #define QMK_BUILDDATE "{current_time}"
+#define QMK_GIT_HASH  "{git_qmk_hash}{'*' if git_dirty else ''}"
 #define CHIBIOS_VERSION "{chibios_version}"
 #define CHIBIOS_CONTRIB_VERSION "{chibios_contrib_version}"
-""")
+"""
+    )
 
     # Show the results
     dump_lines(cli.args.output, version_h_lines, cli.args.quiet)
diff --git a/lib/python/qmk/git.py b/lib/python/qmk/git.py
index 7fa0306f5c..b6c11edbfe 100644
--- a/lib/python/qmk/git.py
+++ b/lib/python/qmk/git.py
@@ -136,3 +136,11 @@ def git_get_ignored_files(check_dir='.'):
     if invalid.returncode != 0:
         return []
     return invalid.stdout.strip().splitlines()
+
+
+def git_get_qmk_hash():
+    output = cli.run(['git', 'rev-parse', '--short', 'HEAD'])
+    if output.returncode != 0:
+        return None
+
+    return output.stdout.strip()
-- 
cgit v1.2.3


From 1d0b4c8d38794dc019ecb224f2992b4ddfa70839 Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Fri, 10 Feb 2023 21:10:14 +0000
Subject: Tidy up use of keycode range helpers (#19756)

---
 lib/python/qmk/cli/generate/keycodes.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

(limited to 'lib/python/qmk')

diff --git a/lib/python/qmk/cli/generate/keycodes.py b/lib/python/qmk/cli/generate/keycodes.py
index 2ed84cd589..f5c646e0ea 100644
--- a/lib/python/qmk/cli/generate/keycodes.py
+++ b/lib/python/qmk/cli/generate/keycodes.py
@@ -8,6 +8,14 @@ from qmk.path import normpath
 from qmk.keycodes import load_spec
 
 
+def _translate_group(group):
+    """Fix up any issues with badly chosen values
+    """
+    if group == 'modifiers':
+        return 'modifier'
+    return group
+
+
 def _render_key(key):
     width = 7
     if 'S(' in key:
@@ -82,7 +90,7 @@ def _generate_helpers(lines, keycodes):
     for group, codes in temp.items():
         lo = keycodes["keycodes"][f'0x{codes[0]:04X}']['key']
         hi = keycodes["keycodes"][f'0x{codes[1]:04X}']['key']
-        lines.append(f'#define IS_{ group.upper() }_KEYCODE(code) ((code) >= {lo} && (code) <= {hi})')
+        lines.append(f'#define IS_{ _translate_group(group).upper() }_KEYCODE(code) ((code) >= {lo} && (code) <= {hi})')
 
 
 def _generate_aliases(lines, keycodes):
-- 
cgit v1.2.3


From 31378839566006de80e4b979079f3149b3a9f4df Mon Sep 17 00:00:00 2001
From: Jouke Witteveen <j.witteveen@gmail.com>
Date: Fri, 10 Feb 2023 23:39:13 +0100
Subject: Typographic updates to source generation (#19160)

---
 lib/python/qmk/cli/generate/keyboard_c.py | 22 +++++++++++-----------
 lib/python/qmk/cli/generate/keyboard_h.py | 23 +++++++++++------------
 2 files changed, 22 insertions(+), 23 deletions(-)

(limited to 'lib/python/qmk')

diff --git a/lib/python/qmk/cli/generate/keyboard_c.py b/lib/python/qmk/cli/generate/keyboard_c.py
index a9b742f323..9004b41abb 100755
--- a/lib/python/qmk/cli/generate/keyboard_c.py
+++ b/lib/python/qmk/cli/generate/keyboard_c.py
@@ -25,17 +25,17 @@ def _gen_led_config(info_data):
     if not config_type:
         return lines
 
-    matrix = [['NO_LED'] * cols for i in range(rows)]
+    matrix = [['NO_LED'] * cols for _ in range(rows)]
     pos = []
     flags = []
 
-    led_config = info_data[config_type]['layout']
-    for index, item in enumerate(led_config, start=0):
-        if 'matrix' in item:
-            (x, y) = item['matrix']
-            matrix[x][y] = str(index)
-        pos.append(f'{{ {item.get("x", 0)},{item.get("y", 0)} }}')
-        flags.append(str(item.get('flags', 0)))
+    led_layout = info_data[config_type]['layout']
+    for index, led_data in enumerate(led_layout):
+        if 'matrix' in led_data:
+            row, col = led_data['matrix']
+            matrix[row][col] = str(index)
+        pos.append(f'{{{led_data.get("x", 0)}, {led_data.get("y", 0)}}}')
+        flags.append(str(led_data.get('flags', 0)))
 
     if config_type == 'rgb_matrix':
         lines.append('#ifdef RGB_MATRIX_ENABLE')
@@ -47,10 +47,10 @@ def _gen_led_config(info_data):
     lines.append('__attribute__ ((weak)) led_config_t g_led_config = {')
     lines.append('  {')
     for line in matrix:
-        lines.append(f'    {{ {",".join(line)} }},')
+        lines.append(f'    {{ {", ".join(line)} }},')
     lines.append('  },')
-    lines.append(f'  {{ {",".join(pos)} }},')
-    lines.append(f'  {{ {",".join(flags)} }},')
+    lines.append(f'  {{ {", ".join(pos)} }},')
+    lines.append(f'  {{ {", ".join(flags)} }},')
     lines.append('};')
     lines.append('#endif')
 
diff --git a/lib/python/qmk/cli/generate/keyboard_h.py b/lib/python/qmk/cli/generate/keyboard_h.py
index 910bd6a08d..152921bdce 100755
--- a/lib/python/qmk/cli/generate/keyboard_h.py
+++ b/lib/python/qmk/cli/generate/keyboard_h.py
@@ -25,32 +25,31 @@ def _generate_layouts(keyboard):
     row_num = kb_info_json['matrix_size']['rows']
 
     lines = []
-    for layout_name in kb_info_json['layouts']:
-        if kb_info_json['layouts'][layout_name]['c_macro']:
+    for layout_name, layout_data in kb_info_json['layouts'].items():
+        if layout_data['c_macro']:
             continue
 
-        if 'matrix' not in kb_info_json['layouts'][layout_name]['layout'][0]:
-            cli.log.debug(f'{keyboard}/{layout_name}: No matrix data!')
+        if not all('matrix' in key_data for key_data in layout_data['layout']):
+            cli.log.debug(f'{keyboard}/{layout_name}: No or incomplete matrix data!')
             continue
 
         layout_keys = []
-        layout_matrix = [['KC_NO' for i in range(col_num)] for i in range(row_num)]
+        layout_matrix = [['KC_NO'] * col_num for _ in range(row_num)]
 
-        for i, key in enumerate(kb_info_json['layouts'][layout_name]['layout']):
-            row = key['matrix'][0]
-            col = key['matrix'][1]
-            identifier = 'k%s%s' % (ROW_LETTERS[row], COL_LETTERS[col])
+        for index, key_data in enumerate(layout_data['layout']):
+            row, col = key_data['matrix']
+            identifier = f'k{ROW_LETTERS[row]}{COL_LETTERS[col]}'
 
             try:
                 layout_matrix[row][col] = identifier
                 layout_keys.append(identifier)
             except IndexError:
-                key_name = key.get('label', identifier)
-                cli.log.error(f'Matrix data out of bounds for layout {layout_name} at index {i} ({key_name}): [{row}, {col}]')
+                key_name = key_data.get('label', identifier)
+                cli.log.error(f'{keyboard}/{layout_name}: Matrix data out of bounds at index {index} ({key_name}): [{row}, {col}]')
                 return []
 
         lines.append('')
-        lines.append('#define %s(%s) {\\' % (layout_name, ', '.join(layout_keys)))
+        lines.append(f'#define {layout_name}({", ".join(layout_keys)}) {{ \\')
 
         rows = ', \\\n'.join(['\t {' + ', '.join(row) + '}' for row in layout_matrix])
         rows += ' \\'
-- 
cgit v1.2.3


From 99918945149f98cf73f9365b0ad49a2fc07d1152 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Sat, 11 Feb 2023 13:45:51 +1100
Subject: Generate encodermap output from keymap.json. (#18915)

Co-authored-by: Joel Challis <git@zvecr.com>
---
 lib/python/qmk/keymap.py | 181 ++++++++++++++++++++++++++++-------------------
 1 file changed, 107 insertions(+), 74 deletions(-)

(limited to 'lib/python/qmk')

diff --git a/lib/python/qmk/keymap.py b/lib/python/qmk/keymap.py
index ce518e379d..dddf6449a7 100644
--- a/lib/python/qmk/keymap.py
+++ b/lib/python/qmk/keymap.py
@@ -30,9 +30,99 @@ const uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = {
 __KEYMAP_GOES_HERE__
 };
 
+#if defined(ENCODER_ENABLE) && defined(ENCODER_MAP_ENABLE)
+const uint16_t PROGMEM encoder_map[][NUM_ENCODERS][2] = {
+__ENCODER_MAP_GOES_HERE__
+};
+#endif // defined(ENCODER_ENABLE) && defined(ENCODER_MAP_ENABLE)
+
+__MACRO_OUTPUT_GOES_HERE__
+
 """
 
 
+def _generate_keymap_table(keymap_json):
+    lines = []
+    for layer_num, layer in enumerate(keymap_json['layers']):
+        if layer_num != 0:
+            lines[-1] = lines[-1] + ','
+        layer = map(_strip_any, layer)
+        layer_keys = ', '.join(layer)
+        lines.append('\t[%s] = %s(%s)' % (layer_num, keymap_json['layout'], layer_keys))
+    return lines
+
+
+def _generate_encodermap_table(keymap_json):
+    lines = []
+    for layer_num, layer in enumerate(keymap_json['encoders']):
+        if layer_num != 0:
+            lines[-1] = lines[-1] + ','
+        encoder_keycode_txt = ', '.join([f'ENCODER_CCW_CW({_strip_any(e["ccw"])}, {_strip_any(e["cw"])})' for e in layer])
+        lines.append('\t[%s] = {%s}' % (layer_num, encoder_keycode_txt))
+    return lines
+
+
+def _generate_macros_function(keymap_json):
+    macro_txt = [
+        'bool process_record_user(uint16_t keycode, keyrecord_t *record) {',
+        '    if (record->event.pressed) {',
+        '        switch (keycode) {',
+    ]
+
+    for i, macro_array in enumerate(keymap_json['macros']):
+        macro = []
+
+        for macro_fragment in macro_array:
+            if isinstance(macro_fragment, str):
+                macro_fragment = macro_fragment.replace('\\', '\\\\')
+                macro_fragment = macro_fragment.replace('\r\n', r'\n')
+                macro_fragment = macro_fragment.replace('\n', r'\n')
+                macro_fragment = macro_fragment.replace('\r', r'\n')
+                macro_fragment = macro_fragment.replace('\t', r'\t')
+                macro_fragment = macro_fragment.replace('"', r'\"')
+
+                macro.append(f'"{macro_fragment}"')
+
+            elif isinstance(macro_fragment, dict):
+                newstring = []
+
+                if macro_fragment['action'] == 'delay':
+                    newstring.append(f"SS_DELAY({macro_fragment['duration']})")
+
+                elif macro_fragment['action'] == 'beep':
+                    newstring.append(r'"\a"')
+
+                elif macro_fragment['action'] == 'tap' and len(macro_fragment['keycodes']) > 1:
+                    last_keycode = macro_fragment['keycodes'].pop()
+
+                    for keycode in macro_fragment['keycodes']:
+                        newstring.append(f'SS_DOWN(X_{keycode})')
+
+                    newstring.append(f'SS_TAP(X_{last_keycode})')
+
+                    for keycode in reversed(macro_fragment['keycodes']):
+                        newstring.append(f'SS_UP(X_{keycode})')
+
+                else:
+                    for keycode in macro_fragment['keycodes']:
+                        newstring.append(f"SS_{macro_fragment['action'].upper()}(X_{keycode})")
+
+                macro.append(''.join(newstring))
+
+        new_macro = "".join(macro)
+        new_macro = new_macro.replace('""', '')
+        macro_txt.append(f'            case QK_MACRO_{i}:')
+        macro_txt.append(f'                SEND_STRING({new_macro});')
+        macro_txt.append('                return false;')
+
+    macro_txt.append('        }')
+    macro_txt.append('    }')
+    macro_txt.append('\n    return true;')
+    macro_txt.append('};')
+    macro_txt.append('')
+    return macro_txt
+
+
 def template_json(keyboard):
     """Returns a `keymap.json` template for a keyboard.
 
@@ -206,83 +296,26 @@ def generate_c(keymap_json):
             A sequence of strings containing macros to implement for this keyboard.
     """
     new_keymap = template_c(keymap_json['keyboard'])
-    layer_txt = []
-
-    for layer_num, layer in enumerate(keymap_json['layers']):
-        if layer_num != 0:
-            layer_txt[-1] = layer_txt[-1] + ','
-        layer = map(_strip_any, layer)
-        layer_keys = ', '.join(layer)
-        layer_txt.append('\t[%s] = %s(%s)' % (layer_num, keymap_json['layout'], layer_keys))
-
+    layer_txt = _generate_keymap_table(keymap_json)
     keymap = '\n'.join(layer_txt)
     new_keymap = new_keymap.replace('__KEYMAP_GOES_HERE__', keymap)
 
-    if keymap_json.get('macros'):
-        macro_txt = [
-            'bool process_record_user(uint16_t keycode, keyrecord_t *record) {',
-            '    if (record->event.pressed) {',
-            '        switch (keycode) {',
-        ]
-
-        for i, macro_array in enumerate(keymap_json['macros']):
-            macro = []
-
-            for macro_fragment in macro_array:
-                if isinstance(macro_fragment, str):
-                    macro_fragment = macro_fragment.replace('\\', '\\\\')
-                    macro_fragment = macro_fragment.replace('\r\n', r'\n')
-                    macro_fragment = macro_fragment.replace('\n', r'\n')
-                    macro_fragment = macro_fragment.replace('\r', r'\n')
-                    macro_fragment = macro_fragment.replace('\t', r'\t')
-                    macro_fragment = macro_fragment.replace('"', r'\"')
-
-                    macro.append(f'"{macro_fragment}"')
-
-                elif isinstance(macro_fragment, dict):
-                    newstring = []
-
-                    if macro_fragment['action'] == 'delay':
-                        newstring.append(f"SS_DELAY({macro_fragment['duration']})")
-
-                    elif macro_fragment['action'] == 'beep':
-                        newstring.append(r'"\a"')
-
-                    elif macro_fragment['action'] == 'tap' and len(macro_fragment['keycodes']) > 1:
-                        last_keycode = macro_fragment['keycodes'].pop()
-
-                        for keycode in macro_fragment['keycodes']:
-                            newstring.append(f'SS_DOWN(X_{keycode})')
-
-                        newstring.append(f'SS_TAP(X_{last_keycode})')
-
-                        for keycode in reversed(macro_fragment['keycodes']):
-                            newstring.append(f'SS_UP(X_{keycode})')
-
-                    else:
-                        for keycode in macro_fragment['keycodes']:
-                            newstring.append(f"SS_{macro_fragment['action'].upper()}(X_{keycode})")
-
-                    macro.append(''.join(newstring))
-
-            new_macro = "".join(macro)
-            new_macro = new_macro.replace('""', '')
-            macro_txt.append(f'            case QK_MACRO_{i}:')
-            macro_txt.append(f'                SEND_STRING({new_macro});')
-            macro_txt.append('                return false;')
-
-        macro_txt.append('        }')
-        macro_txt.append('    }')
-        macro_txt.append('\n    return true;')
-        macro_txt.append('};')
-        macro_txt.append('')
-
-        new_keymap = '\n'.join((new_keymap, *macro_txt))
-
-    if keymap_json.get('host_language'):
-        new_keymap = new_keymap.replace('__INCLUDES__', f'#include "keymap_{keymap_json["host_language"]}.h"\n#include "sendstring_{keymap_json["host_language"]}.h"\n')
-    else:
-        new_keymap = new_keymap.replace('__INCLUDES__', '')
+    encodermap = ''
+    if 'encoders' in keymap_json and keymap_json['encoders'] is not None:
+        encoder_txt = _generate_encodermap_table(keymap_json)
+        encodermap = '\n'.join(encoder_txt)
+    new_keymap = new_keymap.replace('__ENCODER_MAP_GOES_HERE__', encodermap)
+
+    macros = ''
+    if 'macros' in keymap_json and keymap_json['macros'] is not None:
+        macro_txt = _generate_macros_function(keymap_json)
+        macros = '\n'.join(macro_txt)
+    new_keymap = new_keymap.replace('__MACRO_OUTPUT_GOES_HERE__', macros)
+
+    hostlang = ''
+    if 'host_language' in keymap_json and keymap_json['host_language'] is not None:
+        hostlang = f'#include "keymap_{keymap_json["host_language"]}.h"\n#include "sendstring_{keymap_json["host_language"]}.h"\n'
+    new_keymap = new_keymap.replace('__INCLUDES__', hostlang)
 
     return new_keymap
 
-- 
cgit v1.2.3


From 90f3d6201a3cc58f7bb0c6d8abd8092a43c968d5 Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Sat, 11 Feb 2023 20:36:11 +0000
Subject: Reduce false positives in layout name validation (#19646)

---
 lib/python/qmk/info.py | 39 +++++++++++++++++++++++++++------------
 1 file changed, 27 insertions(+), 12 deletions(-)

(limited to 'lib/python/qmk')

diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py
index e73d825696..152e6ce7b6 100644
--- a/lib/python/qmk/info.py
+++ b/lib/python/qmk/info.py
@@ -1,9 +1,10 @@
 """Functions that help us generate and use info.json files.
 """
+import re
 from pathlib import Path
-
 import jsonschema
 from dotty_dict import dotty
+
 from milc import cli
 
 from qmk.constants import CHIBIOS_PROCESSORS, LUFA_PROCESSORS, VUSB_PROCESSORS
@@ -17,15 +18,30 @@ from qmk.math import compute
 true_values = ['1', 'on', 'yes']
 false_values = ['0', 'off', 'no']
 
-# TODO: reduce this list down
-SAFE_LAYOUT_TOKENS = {
-    'ansi',
-    'iso',
-    'wkl',
-    'tkl',
-    'preonic',
-    'planck',
-}
+
+def _keyboard_in_layout_name(keyboard, layout):
+    """Validate that a layout macro does not contain name of keyboard
+    """
+    # TODO: reduce this list down
+    safe_layout_tokens = {
+        'ansi',
+        'iso',
+        'jp',
+        'jis',
+        'ortho',
+        'wkl',
+        'tkl',
+        'preonic',
+        'planck',
+    }
+
+    # Ignore tokens like 'split_3x7_4' or just '2x4'
+    layout = re.sub(r"_split_\d+x\d+_\d+", '', layout)
+    layout = re.sub(r"_\d+x\d+", '', layout)
+
+    name_fragments = set(keyboard.split('/')) - safe_layout_tokens
+
+    return any(fragment in layout for fragment in name_fragments)
 
 
 def _valid_community_layout(layout):
@@ -60,10 +76,9 @@ def _validate(keyboard, info_data):
         _log_warning(info_data, '"LAYOUT_all" should be "LAYOUT" unless additional layouts are provided.')
 
     # Extended layout name checks - ignoring community_layouts and "safe" values
-    name_fragments = set(keyboard.split('/')) - SAFE_LAYOUT_TOKENS
     potential_layouts = set(layouts.keys()) - set(community_layouts_names)
     for layout in potential_layouts:
-        if any(fragment in layout for fragment in name_fragments):
+        if _keyboard_in_layout_name(keyboard, layout):
             _log_warning(info_data, f'Layout "{layout}" should not contain name of keyboard.')
 
     # Filter out any non-existing community layouts
-- 
cgit v1.2.3


From 6ceff1367d1d3c6fbf6f903ea9ef7aff9099eec5 Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Sun, 12 Feb 2023 17:09:34 +0000
Subject: Tidy up use of keycode range helpers (#19813)

---
 lib/python/qmk/cli/generate/keycodes.py | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'lib/python/qmk')

diff --git a/lib/python/qmk/cli/generate/keycodes.py b/lib/python/qmk/cli/generate/keycodes.py
index f5c646e0ea..17503bac63 100644
--- a/lib/python/qmk/cli/generate/keycodes.py
+++ b/lib/python/qmk/cli/generate/keycodes.py
@@ -13,6 +13,8 @@ def _translate_group(group):
     """
     if group == 'modifiers':
         return 'modifier'
+    if group == 'media':
+        return 'consumer'
     return group
 
 
-- 
cgit v1.2.3


From 1283863c0e423eabfc01a4d3906cefbbf94a512c Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Sun, 19 Feb 2023 13:04:50 +1100
Subject: Add `mass-compile` ability to filter by key existence. (#19885)

---
 lib/python/qmk/cli/mass_compile.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

(limited to 'lib/python/qmk')

diff --git a/lib/python/qmk/cli/mass_compile.py b/lib/python/qmk/cli/mass_compile.py
index d8dda0cac3..2821a60c87 100755
--- a/lib/python/qmk/cli/mass_compile.py
+++ b/lib/python/qmk/cli/mass_compile.py
@@ -60,7 +60,7 @@ def _load_keymap_info(keyboard, keymap):
     action='append',
     default=[],
     help=  # noqa: `format-python` and `pytest` don't agree here.
-    "Filter the list of keyboards based on the supplied value in rules.mk. Matches info.json structure, and accepts the format 'features.rgblight=true'. May be passed multiple times, all filters need to match. Value may include wildcards such as '*' and '?'."  # noqa: `format-python` and `pytest` don't agree here.
+    "Filter the list of keyboards based on the supplied value in rules.mk. Matches info.json structure, and accepts the formats 'features.rgblight=true' or 'exists(matrix_pins.direct)'. May be passed multiple times, all filters need to match. Value may include wildcards such as '*' and '?'."  # noqa: `format-python` and `pytest` don't agree here.
 )
 @cli.argument('-km', '--keymap', type=str, default='default', help="The keymap name to build. Default is 'default'.")
 @cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")
@@ -95,9 +95,10 @@ def mass_compile(cli):
             cli.log.info('Parsing data for all matching keyboard/keymap combinations...')
             valid_keymaps = [(e[0], e[1], dotty(e[2])) for e in pool.starmap(_load_keymap_info, target_list)]
 
-            filter_re = re.compile(r'^(?P<key>[a-zA-Z0-9_\.]+)\s*=\s*(?P<value>[^#]+)$')
+            equals_re = re.compile(r'^(?P<key>[a-zA-Z0-9_\.]+)\s*=\s*(?P<value>[^#]+)$')
+            exists_re = re.compile(r'^exists\((?P<key>[a-zA-Z0-9_\.]+)\)$')
             for filter_txt in cli.args.filter:
-                f = filter_re.match(filter_txt)
+                f = equals_re.match(filter_txt)
                 if f is not None:
                     key = f.group('key')
                     value = f.group('value')
@@ -116,6 +117,12 @@ def mass_compile(cli):
 
                     valid_keymaps = filter(_make_filter(key, value), valid_keymaps)
 
+                f = exists_re.match(filter_txt)
+                if f is not None:
+                    key = f.group('key')
+                    cli.log.info(f'Filtering on condition (exists: "{key}")...')
+                    valid_keymaps = filter(lambda e: e[2].get(key) is not None, valid_keymaps)
+
             targets = [(e[0], e[1]) for e in valid_keymaps]
 
     if len(targets) == 0:
-- 
cgit v1.2.3


From 9f2cd9119f18deb824ef7840c69f97c635b485cd Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Wed, 22 Feb 2023 22:50:09 +0000
Subject: Reallocate user/kb keycode ranges (#19907)

---
 lib/python/qmk/json_schema.py | 3 +++
 lib/python/qmk/keycodes.py    | 1 +
 2 files changed, 4 insertions(+)

(limited to 'lib/python/qmk')

diff --git a/lib/python/qmk/json_schema.py b/lib/python/qmk/json_schema.py
index c886a0d868..b00df749cc 100644
--- a/lib/python/qmk/json_schema.py
+++ b/lib/python/qmk/json_schema.py
@@ -107,6 +107,7 @@ def deep_update(origdict, newdict):
 def merge_ordered_dicts(dicts):
     """Merges nested OrderedDict objects resulting from reading a hjson file.
     Later input dicts overrides earlier dicts for plain values.
+    If any value is "!delete!", the existing value will be removed from its parent.
     Arrays will be appended. If the first entry of an array is "!reset!", the contents of the array will be cleared and replaced with RHS.
     Dictionaries will be recursively merged. If any entry is "!reset!", the contents of the dictionary will be cleared and replaced with RHS.
     """
@@ -125,6 +126,8 @@ def merge_ordered_dicts(dicts):
                 target[k] = v[1:]
             else:
                 target[k] = target[k] + v
+        elif v == "!delete!" and isinstance(target, (OrderedDict, dict)):
+            del target[k]
         else:
             target[k] = v
 
diff --git a/lib/python/qmk/keycodes.py b/lib/python/qmk/keycodes.py
index d2f2492829..966930547c 100644
--- a/lib/python/qmk/keycodes.py
+++ b/lib/python/qmk/keycodes.py
@@ -90,6 +90,7 @@ def load_spec(version, lang=None):
 
     # Sort?
     spec['keycodes'] = dict(sorted(spec.get('keycodes', {}).items()))
+    spec['ranges'] = dict(sorted(spec.get('ranges', {}).items()))
 
     # Validate?
     _validate(spec)
-- 
cgit v1.2.3