Merge remote-tracking branch 'upstream/develop'

21 files changed, 796 insertions, 376 deletions

diff --git a/lib/python/qmk/c_parse.py b/lib/python/qmk/c_parse.py
index c14eb490fa..3d73e66091 100644
--- a/lib/python/qmk/c_parse.py
+++ b/lib/python/qmk/c_parse.py
@@ -216,9 +216,9 @@ def _validate_led_config(matrix, matrix_rows, matrix_indexes, position, position
     if len(matrix) != matrix_rows and len(matrix) != (matrix_rows / 2):
         raise ValueError("Unable to parse g_led_config matrix data")
     if len(position) != len(flags):
-        raise ValueError("Unable to parse g_led_config position data")
+        raise ValueError(f"Number of g_led_config physical positions ({len(position)}) does not match number of flags ({len(flags)})")
     if len(matrix_indexes) and (max(matrix_indexes) >= len(flags)):
-        raise ValueError("OOB within g_led_config matrix data")
+        raise ValueError(f"LED index {max(matrix_indexes)} is OOB in g_led_config - should be < {len(flags)}")
     if not all(isinstance(n, int) for n in matrix_indexes):
         raise ValueError("matrix indexes are not all ints")
     if (len(position_raw) % 2) != 0:
diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py
index 98e212c47b..9190af4e50 100644
--- a/lib/python/qmk/cli/__init__.py
+++ b/lib/python/qmk/cli/__init__.py
@@ -47,6 +47,7 @@ subcommands = [
     'qmk.cli.format.python',
     'qmk.cli.format.text',
     'qmk.cli.generate.api',
+    'qmk.cli.generate.autocorrect_data',
     'qmk.cli.generate.compilation_database',
     'qmk.cli.generate.config_h',
     'qmk.cli.generate.develop_pr_list',
@@ -55,7 +56,7 @@ subcommands = [
     'qmk.cli.generate.info_json',
     'qmk.cli.generate.keyboard_c',
     'qmk.cli.generate.keyboard_h',
-    'qmk.cli.generate.layouts',
+    'qmk.cli.generate.keycodes',
     'qmk.cli.generate.rgb_breathe_table',
     'qmk.cli.generate.rules_mk',
     'qmk.cli.generate.version_h',
diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py
index 95118e6687..9e7629906f 100755
--- a/lib/python/qmk/cli/compile.py
+++ b/lib/python/qmk/cli/compile.py
@@ -2,14 +2,13 @@
 
 You can compile a keymap already in the repo or using a QMK Configurator export.
 """
-from subprocess import DEVNULL
-
 from argcomplete.completers import FilesCompleter
+
 from milc import cli
 
 import qmk.path
 from qmk.decorators import automagic_keyboard, automagic_keymap
-from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json
+from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json, build_environment
 from qmk.keyboard import keyboard_completer, keyboard_folder
 from qmk.keymap import keymap_completer
 
@@ -31,48 +30,32 @@ def compile(cli):
 
     If a keyboard and keymap are provided this command will build a firmware based on that.
     """
-    if cli.args.clean and not cli.args.filename and not cli.args.dry_run:
-        if cli.config.compile.keyboard and cli.config.compile.keymap:
-            command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, 'clean')
-            cli.run(command, capture_output=False, stdin=DEVNULL)
-
     # Build the environment vars
-    envs = {}
-    for env in cli.args.env:
-        if '=' in env:
-            key, value = env.split('=', 1)
-            envs[key] = value
-        else:
-            cli.log.warning('Invalid environment variable: %s', env)
+    envs = build_environment(cli.args.env)
 
     # Determine the compile command
-    command = None
+    commands = []
 
     if cli.args.filename:
         # If a configurator JSON was provided generate a keymap and compile it
         user_keymap = parse_configurator_json(cli.args.filename)
-        command = compile_configurator_json(user_keymap, parallel=cli.config.compile.parallel, **envs)
+        commands = [compile_configurator_json(user_keymap, parallel=cli.config.compile.parallel, clean=cli.args.clean, **envs)]
 
-    else:
-        if cli.config.compile.keyboard and cli.config.compile.keymap:
-            # Generate the make command for a specific keyboard/keymap.
-            command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, parallel=cli.config.compile.parallel, **envs)
+    elif cli.config.compile.keyboard and cli.config.compile.keymap:
+        # Generate the make command for a specific keyboard/keymap.
+        if cli.args.clean:
+            commands.append(create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, 'clean', **envs))
+        commands.append(create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, parallel=cli.config.compile.parallel, **envs))
 
-        elif not cli.config.compile.keyboard:
-            cli.log.error('Could not determine keyboard!')
-        elif not cli.config.compile.keymap:
-            cli.log.error('Could not determine keymap!')
-
-    # Compile the firmware, if we're able to
-    if command:
-        cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command))
-        if not cli.args.dry_run:
-            cli.echo('\n')
-            # FIXME(skullydazed/anyone): Remove text=False once milc 1.0.11 has had enough time to be installed everywhere.
-            compile = cli.run(command, capture_output=False, text=False)
-            return compile.returncode
-
-    else:
+    if not commands:
         cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
-        cli.echo('usage: qmk compile [-h] [-b] [-kb KEYBOARD] [-km KEYMAP] [filename]')
+        cli.print_help()
         return False
+
+    cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(commands[-1]))
+    if not cli.args.dry_run:
+        cli.echo('\n')
+        for command in commands:
+            ret = cli.run(command, capture_output=False)
+            if ret.returncode:
+                return ret.returncode
diff --git a/lib/python/qmk/cli/flash.py b/lib/python/qmk/cli/flash.py
index c39f4b36d4..40bfbdab56 100644
--- a/lib/python/qmk/cli/flash.py
+++ b/lib/python/qmk/cli/flash.py
@@ -3,15 +3,13 @@
 You can compile a keymap already in the repo or using a QMK Configurator export.
 A bootloader must be specified.
 """
-from subprocess import DEVNULL
-import sys
-
 from argcomplete.completers import FilesCompleter
+
 from milc import cli
 
 import qmk.path
 from qmk.decorators import automagic_keyboard, automagic_keymap
-from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json
+from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json, build_environment
 from qmk.keyboard import keyboard_completer, keyboard_folder
 from qmk.flashers import flasher
 
@@ -75,59 +73,40 @@ def flash(cli):
                 return False
         except KeyboardInterrupt:
             cli.log.info('Ctrl-C was pressed, exiting...')
-            sys.exit(0)
-
-    else:
-        if cli.args.clean and not cli.args.filename and not cli.args.dry_run:
-            if cli.config.flash.keyboard and cli.config.flash.keymap:
-                command = create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, 'clean')
-                cli.run(command, capture_output=False, stdin=DEVNULL)
-
-        # Build the environment vars
-        envs = {}
-        for env in cli.args.env:
-            if '=' in env:
-                key, value = env.split('=', 1)
-                envs[key] = value
-            else:
-                cli.log.warning('Invalid environment variable: %s', env)
-
-        # Determine the compile command
-        command = ''
-
-        if cli.args.bootloaders:
-            # Provide usage and list bootloaders
-            cli.echo('usage: qmk flash [-h] [-b] [-n] [-kb KEYBOARD] [-km KEYMAP] [-bl BOOTLOADER] [filename]')
-            print_bootloader_help()
-            return False
-
-        if cli.args.filename:
-            # Handle compiling a configurator JSON
-            user_keymap = parse_configurator_json(cli.args.filename)
-            keymap_path = qmk.path.keymap(user_keymap['keyboard'])
-            command = compile_configurator_json(user_keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, **envs)
-
-            cli.log.info('Wrote keymap to {fg_cyan}%s/%s/keymap.c', keymap_path, user_keymap['keymap'])
-
-        else:
-            if cli.config.flash.keyboard and cli.config.flash.keymap:
-                # Generate the make command for a specific keyboard/keymap.
-                command = create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, **envs)
-
-            elif not cli.config.flash.keyboard:
-                cli.log.error('Could not determine keyboard!')
-            elif not cli.config.flash.keymap:
-                cli.log.error('Could not determine keymap!')
-
-        # Compile the firmware, if we're able to
-        if command:
-            cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command))
-            if not cli.args.dry_run:
-                cli.echo('\n')
-                compile = cli.run(command, capture_output=False, stdin=DEVNULL)
-                return compile.returncode
-
-        else:
-            cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
-            cli.echo('usage: qmk flash [-h] [-b] [-n] [-kb KEYBOARD] [-km KEYMAP] [-bl BOOTLOADER] [filename]')
-            return False
+        return True
+
+    if cli.args.bootloaders:
+        # Provide usage and list bootloaders
+        cli.print_help()
+        print_bootloader_help()
+        return False
+
+    # Build the environment vars
+    envs = build_environment(cli.args.env)
+
+    # Determine the compile command
+    commands = []
+
+    if cli.args.filename:
+        # If a configurator JSON was provided generate a keymap and compile it
+        user_keymap = parse_configurator_json(cli.args.filename)
+        commands = [compile_configurator_json(user_keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, clean=cli.args.clean, **envs)]
+
+    elif cli.config.flash.keyboard and cli.config.flash.keymap:
+        # Generate the make command for a specific keyboard/keymap.
+        if cli.args.clean:
+            commands.append(create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, 'clean', **envs))
+        commands.append(create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, **envs))
+
+    if not commands:
+        cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
+        cli.print_help()
+        return False
+
+    cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(commands[-1]))
+    if not cli.args.dry_run:
+        cli.echo('\n')
+        for command in commands:
+            ret = cli.run(command, capture_output=False)
+            if ret.returncode:
+                return ret.returncode
diff --git a/lib/python/qmk/cli/generate/api.py b/lib/python/qmk/cli/generate/api.py
index 8d8ca3cd41..8650a36b84 100755
--- a/lib/python/qmk/cli/generate/api.py
+++ b/lib/python/qmk/cli/generate/api.py
@@ -11,12 +11,57 @@ from qmk.info import info_json
 from qmk.json_encoders import InfoJSONEncoder
 from qmk.json_schema import json_load
 from qmk.keyboard import find_readme, list_keyboards
+from qmk.keycodes import load_spec, list_versions
 
 DATA_PATH = Path('data')
 TEMPLATE_PATH = DATA_PATH / 'templates/api/'
 BUILD_API_PATH = Path('.build/api_data/')
 
 
+def _list_constants(output_folder):
+    """Produce a map of available constants
+    """
+    ret = {}
+    for file in (output_folder / 'constants').glob('**/*_[0-9].[0-9].[0-9].json'):
+        name, version = file.stem.rsplit('_', 1)
+        if name not in ret:
+            ret[name] = []
+        ret[name].append(version)
+
+    # Ensure content is sorted
+    for name in ret:
+        ret[name] = sorted(ret[name])
+
+    return ret
+
+
+def _resolve_keycode_specs(output_folder):
+    """To make it easier for consumers, publish pre-merged spec files
+    """
+    for version in list_versions():
+        overall = load_spec(version)
+
+        output_file = output_folder / f'constants/keycodes_{version}.json'
+        output_file.write_text(json.dumps(overall, indent=4), encoding='utf-8')
+
+    # Purge files consumed by 'load_spec'
+    shutil.rmtree(output_folder / 'constants/keycodes/')
+
+
+def _filtered_copy(src, dst):
+    src = Path(src)
+    dst = Path(dst)
+
+    if dst.suffix == '.hjson':
+        data = json_load(src)
+
+        dst = dst.with_suffix('.json')
+        dst.write_text(json.dumps(data, indent=4), encoding='utf-8')
+        return dst
+
+    return shutil.copy2(src, dst)
+
+
 def _filtered_keyboard_list():
     """Perform basic filtering of list_keyboards
     """
@@ -41,13 +86,14 @@ def generate_api(cli):
     keyboard_list_file = v1_dir / 'keyboard_list.json'  # A simple list of keyboard targets
     keyboard_aliases_file = v1_dir / 'keyboard_aliases.json'  # A list of historical keyboard names and their new name
     keyboard_metadata_file = v1_dir / 'keyboard_metadata.json'  # All the data configurator/via needs for initialization
+    constants_metadata_file = v1_dir / 'constants_metadata.json'  # Metadata for available constants
     usb_file = v1_dir / 'usb.json'  # A mapping of USB VID/PID -> keyboard target
 
     if BUILD_API_PATH.exists():
         shutil.rmtree(BUILD_API_PATH)
 
     shutil.copytree(TEMPLATE_PATH, BUILD_API_PATH)
-    shutil.copytree(DATA_PATH, v1_dir)
+    shutil.copytree(DATA_PATH, v1_dir, copy_function=_filtered_copy)
 
     # Filter down when required
     keyboard_list = _filtered_keyboard_list()
@@ -87,7 +133,7 @@ def generate_api(cli):
 
     # Generate data for the global files
     keyboard_list = sorted(kb_all)
-    keyboard_aliases = json_load(Path('data/mappings/keyboard_aliases.json'))
+    keyboard_aliases = json_load(Path('data/mappings/keyboard_aliases.hjson'))
     keyboard_metadata = {
         'last_updated': current_datetime(),
         'keyboards': keyboard_list,
@@ -95,12 +141,16 @@ def generate_api(cli):
         'usb': usb_list,
     }
 
+    # Feature specific handling
+    _resolve_keycode_specs(v1_dir)
+
     # Write the global JSON files
     keyboard_all_json = json.dumps({'last_updated': current_datetime(), 'keyboards': kb_all}, cls=InfoJSONEncoder)
     usb_json = json.dumps({'last_updated': current_datetime(), 'usb': usb_list}, cls=InfoJSONEncoder)
     keyboard_list_json = json.dumps({'last_updated': current_datetime(), 'keyboards': keyboard_list}, cls=InfoJSONEncoder)
     keyboard_aliases_json = json.dumps({'last_updated': current_datetime(), 'keyboard_aliases': keyboard_aliases}, cls=InfoJSONEncoder)
     keyboard_metadata_json = json.dumps(keyboard_metadata, cls=InfoJSONEncoder)
+    constants_metadata_json = json.dumps({'last_updated': current_datetime(), 'constants': _list_constants(v1_dir)})
 
     if not cli.args.dry_run:
         keyboard_all_file.write_text(keyboard_all_json)
@@ -108,3 +158,4 @@ def generate_api(cli):
         keyboard_list_file.write_text(keyboard_list_json)
         keyboard_aliases_file.write_text(keyboard_aliases_json)
         keyboard_metadata_file.write_text(keyboard_metadata_json)
+        constants_metadata_file.write_text(constants_metadata_json)
diff --git a/lib/python/qmk/cli/generate/autocorrect_data.py b/lib/python/qmk/cli/generate/autocorrect_data.py
new file mode 100644
index 0000000000..00ab6180ab
--- /dev/null
+++ b/lib/python/qmk/cli/generate/autocorrect_data.py
@@ -0,0 +1,289 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Python program to make autocorrect_data.h.
+This program reads from a prepared dictionary file and generates a C source file
+"autocorrect_data.h" with a serialized trie embedded as an array. Run this
+program and pass it as the first argument like:
+$ qmk generate-autocorrect-data autocorrect_dict.txt
+Each line of the dict file defines one typo and its correction with the syntax
+"typo -> correction". Blank lines or lines starting with '#' are ignored.
+Example:
+  :thier        -> their
+  fitler        -> filter
+  lenght        -> length
+  ouput         -> output
+  widht         -> width
+For full documentation, see QMK Docs
+"""
+
+import sys
+import textwrap
+from typing import Any, Dict, Iterator, List, Tuple
+
+from milc import cli
+
+import qmk.path
+from qmk.keyboard import keyboard_completer, keyboard_folder
+from qmk.keymap import keymap_completer, locate_keymap
+
+KC_A = 4
+KC_SPC = 0x2c
+KC_QUOT = 0x34
+
+TYPO_CHARS = dict([
+    ("'", KC_QUOT),
+    (':', KC_SPC),  # "Word break" character.
+] + [(chr(c), c + KC_A - ord('a')) for c in range(ord('a'),
+                                                  ord('z') + 1)])  # Characters a-z.
+
+
+def parse_file(file_name: str) -> List[Tuple[str, str]]:
+    """Parses autocorrections dictionary file.
+  Each line of the file defines one typo and its correction with the syntax
+  "typo -> correction". Blank lines or lines starting with '#' are ignored. The
+  function validates that typos only have characters a-z and that typos are not
+  substrings of other typos, otherwise the longer typo would never trigger.
+  Args:
+    file_name: String, path of the autocorrections dictionary.
+  Returns:
+    List of (typo, correction) tuples.
+  """
+
+    try:
+        from english_words import english_words_lower_alpha_set as correct_words
+    except ImportError:
+        cli.echo('Autocorrection will falsely trigger when a typo is a substring of a correctly spelled word.')
+        cli.echo('To check for this, install the english_words package and rerun this script:')
+        cli.echo('  {fg_cyan}python3 -m pip install english_words')
+        # Use a minimal word list as a fallback.
+        correct_words = ('information', 'available', 'international', 'language', 'loosest', 'reference', 'wealthier', 'entertainment', 'association', 'provides', 'technology', 'statehood')
+
+    autocorrections = []
+    typos = set()
+    for line_number, typo, correction in parse_file_lines(file_name):
+        if typo in typos:
+            cli.log.warning('{fg_red}Error:%d:{fg_reset} Ignoring duplicate typo: "{fg_cyan}%s{fg_reset}"', line_number, typo)
+            continue
+
+        # Check that `typo` is valid.
+        if not (all([c in TYPO_CHARS for c in typo])):
+            cli.log.error('{fg_red}Error:%d:{fg_reset} Typo "{fg_cyan}%s{fg_reset}" has characters other than a-z, \' and :.', line_number, typo)
+            sys.exit(1)
+        for other_typo in typos:
+            if typo in other_typo or other_typo in typo:
+                cli.log.error('{fg_red}Error:%d:{fg_reset} Typos may not be substrings of one another, otherwise the longer typo would never trigger: "{fg_cyan}%s{fg_reset}" vs. "{fg_cyan}%s{fg_reset}".', line_number, typo, other_typo)
+                sys.exit(1)
+        if len(typo) < 5:
+            cli.log.warning('{fg_yellow}Warning:%d:{fg_reset} It is suggested that typos are at least 5 characters long to avoid false triggers: "{fg_cyan}%s{fg_reset}"', line_number, typo)
+        if len(typo) > 127:
+            cli.log.error('{fg_red}Error:%d:{fg_reset} Typo exceeds 127 chars: "{fg_cyan}%s{fg_reset}"', line_number, typo)
+            sys.exit(1)
+
+        check_typo_against_dictionary(typo, line_number, correct_words)
+
+        autocorrections.append((typo, correction))
+        typos.add(typo)
+
+    return autocorrections
+
+
+def make_trie(autocorrections: List[Tuple[str, str]]) -> Dict[str, Any]:
+    """Makes a trie from the the typos, writing in reverse.
+  Args:
+    autocorrections: List of (typo, correction) tuples.
+  Returns:
+    Dict of dict, representing the trie.
+  """
+    trie = {}
+    for typo, correction in autocorrections:
+        node = trie
+        for letter in typo[::-1]:
+            node = node.setdefault(letter, {})
+        node['LEAF'] = (typo, correction)
+
+    return trie
+
+
+def parse_file_lines(file_name: str) -> Iterator[Tuple[int, str, str]]:
+    """Parses lines read from `file_name` into typo-correction pairs."""
+
+    line_number = 0
+    for line in open(file_name, 'rt'):
+        line_number += 1
+        line = line.strip()
+        if line and line[0] != '#':
+            # Parse syntax "typo -> correction", using strip to ignore indenting.
+            tokens = [token.strip() for token in line.split('->', 1)]
+            if len(tokens) != 2 or not tokens[0]:
+                print(f'Error:{line_number}: Invalid syntax: "{line}"')
+                sys.exit(1)
+
+            typo, correction = tokens
+            typo = typo.lower()  # Force typos to lowercase.
+            typo = typo.replace(' ', ':')
+
+            yield line_number, typo, correction
+
+
+def check_typo_against_dictionary(typo: str, line_number: int, correct_words) -> None:
+    """Checks `typo` against English dictionary words."""
+
+    if typo.startswith(':') and typo.endswith(':'):
+        if typo[1:-1] in correct_words:
+            cli.log.warning('{fg_yellow}Warning:%d:{fg_reset} Typo "{fg_cyan}%s{fg_reset}" is a correctly spelled dictionary word.', line_number, typo)
+    elif typo.startswith(':') and not typo.endswith(':'):
+        for word in correct_words:
+            if word.startswith(typo[1:]):
+                cli.log.warning('{fg_yellow}Warning:%d: {fg_reset}Typo "{fg_cyan}%s{fg_reset}" would falsely trigger on correctly spelled word "{fg_cyan}%s{fg_reset}".', line_number, typo, word)
+    elif not typo.startswith(':') and typo.endswith(':'):
+        for word in correct_words:
+            if word.endswith(typo[:-1]):
+                cli.log.warning('{fg_yellow}Warning:%d:{fg_reset} Typo "{fg_cyan}%s{fg_reset}" would falsely trigger on correctly spelled word "{fg_cyan}%s{fg_reset}".', line_number, typo, word)
+    elif not typo.startswith(':') and not typo.endswith(':'):
+        for word in correct_words:
+            if typo in word:
+                cli.log.warning('{fg_yellow}Warning:%d:{fg_reset} Typo "{fg_cyan}%s{fg_reset}" would falsely trigger on correctly spelled word "{fg_cyan}%s{fg_reset}".', line_number, typo, word)
+
+
+def serialize_trie(autocorrections: List[Tuple[str, str]], trie: Dict[str, Any]) -> List[int]:
+    """Serializes trie and correction data in a form readable by the C code.
+  Args:
+    autocorrections: List of (typo, correction) tuples.
+    trie: Dict of dicts.
+  Returns:
+    List of ints in the range 0-255.
+  """
+    table = []
+
+    # Traverse trie in depth first order.
+    def traverse(trie_node):
+        if 'LEAF' in trie_node:  # Handle a leaf trie node.
+            typo, correction = trie_node['LEAF']
+            word_boundary_ending = typo[-1] == ':'
+            typo = typo.strip(':')
+            i = 0  # Make the autocorrection data for this entry and serialize it.
+            while i < min(len(typo), len(correction)) and typo[i] == correction[i]:
+                i += 1
+            backspaces = len(typo) - i - 1 + word_boundary_ending
+            assert 0 <= backspaces <= 63
+            correction = correction[i:]
+            bs_count = [backspaces + 128]
+            data = bs_count + list(bytes(correction, 'ascii')) + [0]
+
+            entry = {'data': data, 'links': [], 'byte_offset': 0}
+            table.append(entry)
+        elif len(trie_node) == 1:  # Handle trie node with a single child.
+            c, trie_node = next(iter(trie_node.items()))
+            entry = {'chars': c, 'byte_offset': 0}
+
+            # It's common for a trie to have long chains of single-child nodes. We
+            # find the whole chain so that we can serialize it more efficiently.
+            while len(trie_node) == 1 and 'LEAF' not in trie_node:
+                c, trie_node = next(iter(trie_node.items()))
+                entry['chars'] += c
+
+            table.append(entry)
+            entry['links'] = [traverse(trie_node)]
+        else:  # Handle trie node with multiple children.
+            entry = {'chars': ''.join(sorted(trie_node.keys())), 'byte_offset': 0}
+            table.append(entry)
+            entry['links'] = [traverse(trie_node[c]) for c in entry['chars']]
+        return entry
+
+    traverse(trie)
+
+    def serialize(e: Dict[str, Any]) -> List[int]:
+        if not e['links']:  # Handle a leaf table entry.
+            return e['data']
+        elif len(e['links']) == 1:  # Handle a chain table entry.
+            return [TYPO_CHARS[c] for c in e['chars']] + [0]  # + encode_link(e['links'][0]))
+        else:  # Handle a branch table entry.
+            data = []
+            for c, link in zip(e['chars'], e['links']):
+                data += [TYPO_CHARS[c] | (0 if data else 64)] + encode_link(link)
+            return data + [0]
+
+    byte_offset = 0
+    for e in table:  # To encode links, first compute byte offset of each entry.
+        e['byte_offset'] = byte_offset
+        byte_offset += len(serialize(e))
+        assert 0 <= byte_offset <= 0xffff
+
+    return [b for e in table for b in serialize(e)]  # Serialize final table.
+
+
+def encode_link(link: Dict[str, Any]) -> List[int]:
+    """Encodes a node link as two bytes."""
+    byte_offset = link['byte_offset']
+    if not (0 <= byte_offset <= 0xffff):
+        cli.log.error('{fg_red}Error:{fg_reset} The autocorrection table is too large, a node link exceeds 64KB limit. Try reducing the autocorrection dict to fewer entries.')
+        sys.exit(1)
+    return [byte_offset & 255, byte_offset >> 8]
+
+
+def write_generated_code(autocorrections: List[Tuple[str, str]], data: List[int], file_name: str) -> None:
+    """Writes autocorrection data as generated C code to `file_name`.
+  Args:
+    autocorrections: List of (typo, correction) tuples.
+    data: List of ints in 0-255, the serialized trie.
+    file_name: String, path of the output C file.
+  """
+    assert all(0 <= b <= 255 for b in data)
+
+    def typo_len(e: Tuple[str, str]) -> int:
+        return len(e[0])
+
+    min_typo = min(autocorrections, key=typo_len)[0]
+    max_typo = max(autocorrections, key=typo_len)[0]
+    generated_code = ''.join([
+        '// Generated code.\n\n', f'// Autocorrection dictionary ({len(autocorrections)} entries):\n', ''.join(sorted(f'//   {typo:<{len(max_typo)}} -> {correction}\n' for typo, correction in autocorrections)),
+        f'\n#define AUTOCORRECT_MIN_LENGTH {len(min_typo)}  // "{min_typo}"\n', f'#define AUTOCORRECT_MAX_LENGTH {len(max_typo)}  // "{max_typo}"\n\n', f'#define DICTIONARY_SIZE {len(data)}\n\n',
+        textwrap.fill('static const uint8_t autocorrect_data[DICTIONARY_SIZE] PROGMEM = {%s};' % (', '.join(map(str, data))), width=120, subsequent_indent='    '), '\n\n'
+    ])
+
+    with open(file_name, 'wt') as f:
+        f.write(generated_code)
+
+
+@cli.argument('filename', default='autocorrect_dict.txt', help='The autocorrection database file')
+@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
+@cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.')
+@cli.argument('-o', '--output', arg_only=True, type=qmk.path.normpath, help='File to write to')
+@cli.subcommand('Generate the autocorrection data file from a dictionary file.')
+def generate_autocorrect_data(cli):
+    autocorrections = parse_file(cli.args.filename)
+    trie = make_trie(autocorrections)
+    data = serialize_trie(autocorrections, trie)
+    # Environment processing
+    if cli.args.output == '-':
+        cli.args.output = None
+
+    if cli.args.output:
+        cli.args.output.parent.mkdir(parents=True, exist_ok=True)
+        cli.log.info('Creating autocorrect database at {fg_cyan}%s', cli.args.output)
+        write_generated_code(autocorrections, data, cli.args.output)
+
+    else:
+        current_keyboard = cli.args.keyboard or cli.config.user.keyboard or cli.config.generate_autocorrect_data.keyboard
+        current_keymap = cli.args.keymap or cli.config.user.keymap or cli.config.generate_autocorrect_data.keymap
+
+        if current_keyboard and current_keymap:
+            filename = locate_keymap(current_keyboard, current_keymap).parent / 'autocorrect_data.h'
+            cli.log.info('Creating autocorrect database at {fg_cyan}%s', filename)
+            write_generated_code(autocorrections, data, filename)
+
+        else:
+            write_generated_code(autocorrections, data, 'autocorrect_data.h')
+
+    cli.log.info('Processed %d autocorrection entries to table with %d bytes.', len(autocorrections), len(data))
diff --git a/lib/python/qmk/cli/generate/config_h.py b/lib/python/qmk/cli/generate/config_h.py
index a26dcdf7d7..31b8d70635 100755
--- a/lib/python/qmk/cli/generate/config_h.py
+++ b/lib/python/qmk/cli/generate/config_h.py
@@ -1,18 +1,27 @@
 """Used by the make system to generate info_config.h from info.json.
 """
 from pathlib import Path
-
 from dotty_dict import dotty
+
+from argcomplete.completers import FilesCompleter
 from milc import cli
 
-from qmk.info import info_json, keymap_json_config
+from qmk.info import info_json
 from qmk.json_schema import json_load
 from qmk.keyboard import keyboard_completer, keyboard_folder
-from qmk.commands import dump_lines
-from qmk.path import normpath
+from qmk.commands import dump_lines, parse_configurator_json
+from qmk.path import normpath, FileType
 from qmk.constants import GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE
 
 
+def generate_define(define, value=None):
+    value = f' {value}' if value is not None else ''
+    return f"""
+#ifndef {define}
+#    define {define}{value}
+#endif // {define}"""
+
+
 def direct_pins(direct_pins, postfix):
     """Return the config.h lines that set the direct pins.
     """
@@ -22,11 +31,7 @@ def direct_pins(direct_pins, postfix):
         cols = ','.join(map(str, [col or 'NO_PIN' for col in row]))
         rows.append('{' + cols + '}')
 
-    return f"""
-#ifndef DIRECT_PINS{postfix}
-#   define DIRECT_PINS{postfix} {{ {", ".join(rows)} }}
-#endif // DIRECT_PINS{postfix}
-"""
+    return generate_define(f'DIRECT_PINS{postfix}', f'{{ {", ".join(rows)} }}')
 
 
 def pin_array(define, pins, postfix):
@@ -34,11 +39,7 @@ def pin_array(define, pins, postfix):
     """
     pin_array = ', '.join(map(str, [pin or 'NO_PIN' for pin in pins]))
 
-    return f"""
-#ifndef {define}_PINS{postfix}
-#   define {define}_PINS{postfix} {{ {pin_array} }}
-#endif // {define}_PINS{postfix}
-"""
+    return generate_define(f'{define}_PINS{postfix}', f'{{ {pin_array} }}')
 
 
 def matrix_pins(matrix_pins, postfix=''):
@@ -62,24 +63,14 @@ def generate_matrix_size(kb_info_json, config_h_lines):
     """Add the matrix size to the config.h.
     """
     if 'matrix_pins' in kb_info_json:
-        col_count = kb_info_json['matrix_size']['cols']
-        row_count = kb_info_json['matrix_size']['rows']
-
-        config_h_lines.append(f"""
-#ifndef MATRIX_COLS
-#   define MATRIX_COLS {col_count}
-#endif // MATRIX_COLS
-
-#ifndef MATRIX_ROWS
-#   define MATRIX_ROWS {row_count}
-#endif // MATRIX_ROWS
-""")
+        config_h_lines.append(generate_define('MATRIX_COLS', kb_info_json['matrix_size']['cols']))
+        config_h_lines.append(generate_define('MATRIX_ROWS', kb_info_json['matrix_size']['rows']))
 
 
 def generate_config_items(kb_info_json, config_h_lines):
     """Iterate through the info_config map to generate basic config values.
     """
-    info_config_map = json_load(Path('data/mappings/info_config.json'))
+    info_config_map = json_load(Path('data/mappings/info_config.hjson'))
 
     for config_key, info_dict in info_config_map.items():
         info_key = info_dict['info_key']
@@ -95,43 +86,23 @@ def generate_config_items(kb_info_json, config_h_lines):
             continue
 
         if key_type.startswith('array.array'):
-            config_h_lines.append('')
-            config_h_lines.append(f'#ifndef {config_key}')
-            config_h_lines.append(f'#   define {config_key} {{ {", ".join(["{" + ",".join(list(map(str, x))) + "}" for x in config_value])} }}')
-            config_h_lines.append(f'#endif // {config_key}')
+            config_h_lines.append(generate_define(config_key, f'{{ {", ".join(["{" + ",".join(list(map(str, x))) + "}" for x in config_value])} }}'))
         elif key_type.startswith('array'):
-            config_h_lines.append('')
-            config_h_lines.append(f'#ifndef {config_key}')
-            config_h_lines.append(f'#   define {config_key} {{ {", ".join(map(str, config_value))} }}')
-            config_h_lines.append(f'#endif // {config_key}')
+            config_h_lines.append(generate_define(config_key, f'{{ {", ".join(map(str, config_value))} }}'))
         elif key_type == 'bool':
             if config_value:
-                config_h_lines.append('')
-                config_h_lines.append(f'#ifndef {config_key}')
-                config_h_lines.append(f'#   define {config_key}')
-                config_h_lines.append(f'#endif // {config_key}')
+                config_h_lines.append(generate_define(config_key))
         elif key_type == 'mapping':
             for key, value in config_value.items():
-                config_h_lines.append('')
-                config_h_lines.append(f'#ifndef {key}')
-                config_h_lines.append(f'#   define {key} {value}')
-                config_h_lines.append(f'#endif // {key}')
+                config_h_lines.append(generate_define(key, value))
         elif key_type == 'str':
-            config_h_lines.append('')
-            config_h_lines.append(f'#ifndef {config_key}')
-            config_h_lines.append(f'#   define {config_key} "{config_value}"')
-            config_h_lines.append(f'#endif // {config_key}')
+            escaped_str = config_value.replace('\\', '\\\\').replace('"', '\\"')
+            config_h_lines.append(generate_define(config_key, f'"{escaped_str}"'))
         elif key_type == 'bcd_version':
             (major, minor, revision) = config_value.split('.')
-            config_h_lines.append('')
-            config_h_lines.append(f'#ifndef {config_key}')
-            config_h_lines.append(f'#   define {config_key} 0x{major.zfill(2)}{minor}{revision}')
-            config_h_lines.append(f'#endif // {config_key}')
+            config_h_lines.append(generate_define(config_key, f'0x{major.zfill(2)}{minor}{revision}'))
         else:
-            config_h_lines.append('')
-            config_h_lines.append(f'#ifndef {config_key}')
-            config_h_lines.append(f'#   define {config_key} {config_value}')
-            config_h_lines.append(f'#endif // {config_key}')
+            config_h_lines.append(generate_define(config_key, config_value))
 
 
 def generate_encoder_config(encoder_json, config_h_lines, postfix=''):
@@ -144,24 +115,15 @@ def generate_encoder_config(encoder_json, config_h_lines, postfix=''):
         b_pads.append(encoder["pin_b"])
         resolutions.append(encoder.get("resolution", None))
 
-    config_h_lines.append(f'#ifndef ENCODERS_PAD_A{postfix}')
-    config_h_lines.append(f'#   define ENCODERS_PAD_A{postfix} {{ { ", ".join(a_pads) } }}')
-    config_h_lines.append(f'#endif // ENCODERS_PAD_A{postfix}')
-
-    config_h_lines.append(f'#ifndef ENCODERS_PAD_B{postfix}')
-    config_h_lines.append(f'#   define ENCODERS_PAD_B{postfix} {{ { ", ".join(b_pads) } }}')
-    config_h_lines.append(f'#endif // ENCODERS_PAD_B{postfix}')
+    config_h_lines.append(generate_define(f'ENCODERS_PAD_A{postfix}', f'{{ {", ".join(a_pads)} }}'))
+    config_h_lines.append(generate_define(f'ENCODERS_PAD_B{postfix}', f'{{ {", ".join(b_pads)} }}'))
 
     if None in resolutions:
         cli.log.debug("Unable to generate ENCODER_RESOLUTION configuration")
     elif len(set(resolutions)) == 1:
-        config_h_lines.append(f'#ifndef ENCODER_RESOLUTION{postfix}')
-        config_h_lines.append(f'#   define ENCODER_RESOLUTION{postfix} { resolutions[0] }')
-        config_h_lines.append(f'#endif // ENCODER_RESOLUTION{postfix}')
+        config_h_lines.append(generate_define(f'ENCODER_RESOLUTION{postfix}', resolutions[0]))
     else:
-        config_h_lines.append(f'#ifndef ENCODER_RESOLUTIONS{postfix}')
-        config_h_lines.append(f'#   define ENCODER_RESOLUTIONS{postfix} {{ { ", ".join(map(str,resolutions)) } }}')
-        config_h_lines.append(f'#endif // ENCODER_RESOLUTIONS{postfix}')
+        config_h_lines.append(generate_define(f'ENCODER_RESOLUTIONS{postfix}', f'{{ {", ".join(map(str,resolutions))} }}'))
 
 
 def generate_split_config(kb_info_json, config_h_lines):
@@ -170,35 +132,23 @@ def generate_split_config(kb_info_json, config_h_lines):
         if kb_info_json['split']['primary'] in ('left', 'right'):
             config_h_lines.append('')
             config_h_lines.append('#ifndef MASTER_LEFT')
-            config_h_lines.append('#   ifndef MASTER_RIGHT')
+            config_h_lines.append('#    ifndef MASTER_RIGHT')
             if kb_info_json['split']['primary'] == 'left':
-                config_h_lines.append('#       define MASTER_LEFT')
+                config_h_lines.append('#        define MASTER_LEFT')
             elif kb_info_json['split']['primary'] == 'right':
-                config_h_lines.append('#       define MASTER_RIGHT')
-            config_h_lines.append('#   endif // MASTER_RIGHT')
+                config_h_lines.append('#        define MASTER_RIGHT')
+            config_h_lines.append('#    endif // MASTER_RIGHT')
             config_h_lines.append('#endif // MASTER_LEFT')
         elif kb_info_json['split']['primary'] == 'pin':
-            config_h_lines.append('')
-            config_h_lines.append('#ifndef SPLIT_HAND_PIN')
-            config_h_lines.append('#   define SPLIT_HAND_PIN')
-            config_h_lines.append('#endif // SPLIT_HAND_PIN')
+            config_h_lines.append(generate_define('SPLIT_HAND_PIN'))
         elif kb_info_json['split']['primary'] == 'matrix_grid':
-            config_h_lines.append('')
-            config_h_lines.append('#ifndef SPLIT_HAND_MATRIX_GRID')
-            config_h_lines.append('#   define SPLIT_HAND_MATRIX_GRID {%s}' % (','.join(kb_info_json["split"]["matrix_grid"],)))
-            config_h_lines.append('#endif // SPLIT_HAND_MATRIX_GRID')
+            config_h_lines.append(generate_define('SPLIT_HAND_MATRIX_GRID', f'{{ {",".join(kb_info_json["split"]["matrix_grid"])} }}'))
         elif kb_info_json['split']['primary'] == 'eeprom':
-            config_h_lines.append('')
-            config_h_lines.append('#ifndef EE_HANDS')
-            config_h_lines.append('#   define EE_HANDS')
-            config_h_lines.append('#endif // EE_HANDS')
+            config_h_lines.append(generate_define('EE_HANDS'))
 
     if 'protocol' in kb_info_json['split'].get('transport', {}):
         if kb_info_json['split']['transport']['protocol'] == 'i2c':
-            config_h_lines.append('')
-            config_h_lines.append('#ifndef USE_I2C')
-            config_h_lines.append('#   define USE_I2C')
-            config_h_lines.append('#endif // USE_I2C')
+            config_h_lines.append(generate_define('USE_I2C'))
 
     if 'right' in kb_info_json['split'].get('matrix_pins', {}):
         config_h_lines.append(matrix_pins(kb_info_json['split']['matrix_pins']['right'], '_RIGHT'))
@@ -207,19 +157,30 @@ def generate_split_config(kb_info_json, config_h_lines):
         generate_encoder_config(kb_info_json['split']['encoder']['right'], config_h_lines, '_RIGHT')
 
 
+def generate_led_animations_config(led_feature_json, config_h_lines, prefix):
+    for animation in led_feature_json.get('animations', {}):
+        if led_feature_json['animations'][animation]:
+            config_h_lines.append(generate_define(f'{prefix}{animation.upper()}'))
+
+
+@cli.argument('filename', nargs='?', arg_only=True, type=FileType('r'), completer=FilesCompleter('.json'), help='A configurator export JSON to be compiled and flashed or a pre-compiled binary firmware file (bin/hex) to be flashed.')
 @cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to')
 @cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages")
-@cli.argument('-kb', '--keyboard', arg_only=True, type=keyboard_folder, completer=keyboard_completer, required=True, help='Keyboard to generate config.h for.')
-@cli.argument('-km', '--keymap', arg_only=True, help='Keymap to generate config.h for.')
+@cli.argument('-kb', '--keyboard', arg_only=True, type=keyboard_folder, completer=keyboard_completer, help='Keyboard to generate config.h for.')
 @cli.subcommand('Used by the make system to generate info_config.h from info.json', hidden=True)
 def generate_config_h(cli):
     """Generates the info_config.h file.
     """
     # Determine our keyboard/keymap
-    if cli.args.keymap:
-        kb_info_json = dotty(keymap_json_config(cli.args.keyboard, cli.args.keymap))
-    else:
+    if cli.args.filename:
+        user_keymap = parse_configurator_json(cli.args.filename)
+        kb_info_json = dotty(user_keymap.get('config', {}))
+    elif cli.args.keyboard:
         kb_info_json = dotty(info_json(cli.args.keyboard))
+    else:
+        cli.log.error('You must supply a configurator export or `--keyboard`.')
+        cli.subcommands['generate-config-h'].print_help()
+        return False
 
     # Build the info_config.h file.
     config_h_lines = [GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE, '#pragma once']
@@ -237,5 +198,14 @@ def generate_config_h(cli):
     if 'split' in kb_info_json:
         generate_split_config(kb_info_json, config_h_lines)
 
+    if 'led_matrix' in kb_info_json:
+        generate_led_animations_config(kb_info_json['led_matrix'], config_h_lines, 'ENABLE_LED_MATRIX_')
+
+    if 'rgb_matrix' in kb_info_json:
+        generate_led_animations_config(kb_info_json['rgb_matrix'], config_h_lines, 'ENABLE_RGB_MATRIX_')
+
+    if 'rgblight' in kb_info_json:
+        generate_led_animations_config(kb_info_json['rgblight'], config_h_lines, 'RGBLIGHT_EFFECT_')
+
     # Show the results
     dump_lines(cli.args.output, config_h_lines, cli.args.quiet)
diff --git a/lib/python/qmk/cli/generate/keyboard_h.py b/lib/python/qmk/cli/generate/keyboard_h.py
index 54ddb4cffd..910bd6a08d 100755
--- a/lib/python/qmk/cli/generate/keyboard_h.py
+++ b/lib/python/qmk/cli/generate/keyboard_h.py
@@ -1,33 +1,72 @@
 """Used by the make system to generate keyboard.h from info.json.
 """
+from pathlib import Path
+
 from milc import cli
 
+from qmk.path import normpath
 from qmk.info import info_json
 from qmk.commands import dump_lines
 from qmk.keyboard import keyboard_completer, keyboard_folder
-from qmk.path import normpath
-from qmk.constants import GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE
+from qmk.constants import COL_LETTERS, ROW_LETTERS, GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE
 
 
-def would_populate_layout_h(keyboard):
-    """Detect if a given keyboard is doing data driven layouts
+def _generate_layouts(keyboard):
+    """Generates the layouts.h file.
     """
     # Build the info.json file
     kb_info_json = info_json(keyboard)
 
+    if 'matrix_size' not in kb_info_json:
+        cli.log.error(f'{keyboard}: Invalid matrix config.')
+        return []
+
+    col_num = kb_info_json['matrix_size']['cols']
+    row_num = kb_info_json['matrix_size']['rows']
+
+    lines = []
     for layout_name in kb_info_json['layouts']:
         if kb_info_json['layouts'][layout_name]['c_macro']:
             continue
 
         if 'matrix' not in kb_info_json['layouts'][layout_name]['layout'][0]:
-            cli.log.debug('%s/%s: No matrix data!', keyboard, layout_name)
+            cli.log.debug(f'{keyboard}/{layout_name}: No matrix data!')
             continue
 
-        return True
+        layout_keys = []
+        layout_matrix = [['KC_NO' for i in range(col_num)] for i in range(row_num)]
+
+        for i, key in enumerate(kb_info_json['layouts'][layout_name]['layout']):
+            row = key['matrix'][0]
+            col = key['matrix'][1]
+            identifier = 'k%s%s' % (ROW_LETTERS[row], COL_LETTERS[col])
 
-    return False
+            try:
+                layout_matrix[row][col] = identifier
+                layout_keys.append(identifier)
+            except IndexError:
+                key_name = key.get('label', identifier)
+                cli.log.error(f'Matrix data out of bounds for layout {layout_name} at index {i} ({key_name}): [{row}, {col}]')
+                return []
 
+        lines.append('')
+        lines.append('#define %s(%s) {\\' % (layout_name, ', '.join(layout_keys)))
 
+        rows = ', \\\n'.join(['\t {' + ', '.join(row) + '}' for row in layout_matrix])
+        rows += ' \\'
+        lines.append(rows)
+        lines.append('}')
+
+    for alias, target in kb_info_json.get('layout_aliases', {}).items():
+        lines.append('')
+        lines.append(f'#ifndef {alias}')
+        lines.append(f'#   define {alias} {target}')
+        lines.append('#endif')
+
+    return lines
+
+
+@cli.argument('-i', '--include', nargs='?', arg_only=True, help='Optional file to include')
 @cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to')
 @cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages")
 @cli.argument('-kb', '--keyboard', arg_only=True, type=keyboard_folder, completer=keyboard_completer, required=True, help='Keyboard to generate keyboard.h for.')
@@ -35,13 +74,23 @@ def would_populate_layout_h(keyboard):
 def generate_keyboard_h(cli):
     """Generates the keyboard.h file.
     """
-    has_layout_h = would_populate_layout_h(cli.args.keyboard)
+    keyboard_h = cli.args.include
+    dd_layouts = _generate_layouts(cli.args.keyboard)
+    valid_config = dd_layouts or keyboard_h
 
     # Build the layouts.h file.
     keyboard_h_lines = [GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE, '#pragma once', '#include "quantum.h"']
 
-    if not has_layout_h:
-        keyboard_h_lines.append('#error("<keyboard>.h is only optional for data driven keyboards - kb.h == bad times")')
+    keyboard_h_lines.append('')
+    keyboard_h_lines.append('// Layout content')
+    if dd_layouts:
+        keyboard_h_lines.extend(dd_layouts)
+    if keyboard_h:
+        keyboard_h_lines.append(f'#include "{Path(keyboard_h).name}"')
+
+    # Protect against poorly configured keyboards
+    if not valid_config:
+        keyboard_h_lines.append('#error("<keyboard>.h is required unless your keyboard uses data-driven configuration. Please rename your keyboard\'s header file to <keyboard>.h")')
 
     # Show the results
     dump_lines(cli.args.output, keyboard_h_lines, cli.args.quiet)
diff --git a/lib/python/qmk/cli/generate/keycodes.py b/lib/python/qmk/cli/generate/keycodes.py
new file mode 100644
index 0000000000..29b7db3c80
--- /dev/null
+++ b/lib/python/qmk/cli/generate/keycodes.py
@@ -0,0 +1,88 @@
+"""Used by the make system to generate keycodes.h from keycodes_{version}.json
+"""
+from milc import cli
+
+from qmk.constants import GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE
+from qmk.commands import dump_lines
+from qmk.path import normpath
+from qmk.keycodes import load_spec
+
+
+def _generate_ranges(lines, keycodes):
+    lines.append('')
+    lines.append('enum qk_keycode_ranges {')
+    lines.append('// Ranges')
+    for key, value in keycodes["ranges"].items():
+        lo, mask = map(lambda x: int(x, 16), key.split("/"))
+        hi = lo + mask
+        define = value.get("define")
+        lines.append(f'    {define.ljust(30)} = 0x{lo:04X},')
+        lines.append(f'    {(define + "_MAX").ljust(30)} = 0x{hi:04X},')
+    lines.append('};')
+
+
+def _generate_defines(lines, keycodes):
+    lines.append('')
+    lines.append('enum qk_keycode_defines {')
+    lines.append('// Keycodes')
+    for key, value in keycodes["keycodes"].items():
+        lines.append(f'    {value.get("key")} = {key},')
+
+    lines.append('')
+    lines.append('// Alias')
+    for key, value in keycodes["keycodes"].items():
+        temp = value.get("key")
+        for alias in value.get("aliases", []):
+            lines.append(f'    {alias.ljust(10)} = {temp},')
+
+    lines.append('};')
+
+
+def _generate_helpers(lines, keycodes):
+    lines.append('')
+    lines.append('// Range Helpers')
+    for value in keycodes["ranges"].values():
+        define = value.get("define")
+        lines.append(f'#define IS_{define}(code) ((code) >= {define} && (code) <= {define + "_MAX"})')
+
+    # extract min/max
+    temp = {}
+    for key, value in keycodes["keycodes"].items():
+        group = value.get('group', None)
+        if not group:
+            continue
+        if group not in temp:
+            temp[group] = [0xFFFF, 0]
+        key = int(key, 16)
+        if key < temp[group][0]:
+            temp[group][0] = key
+        if key > temp[group][1]:
+            temp[group][1] = key
+
+    lines.append('')
+    lines.append('// Group Helpers')
+    for group, codes in temp.items():
+        lo = keycodes["keycodes"][f'0x{codes[0]:04X}']['key']
+        hi = keycodes["keycodes"][f'0x{codes[1]:04X}']['key']
+        lines.append(f'#define IS_{ group.upper() }_KEYCODE(code) ((code) >= {lo} && (code) <= {hi})')
+
+
+@cli.argument('-v', '--version', arg_only=True, required=True, help='Version of keycodes to generate.')
+@cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to')
+@cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages")
+@cli.subcommand('Used by the make system to generate keycodes.h from keycodes_{version}.json', hidden=True)
+def generate_keycodes(cli):
+    """Generates the keycodes.h file.
+    """
+
+    # Build the keycodes.h file.
+    keycodes_h_lines = [GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE, '#pragma once', '// clang-format off']
+
+    keycodes = load_spec(cli.args.version)
+
+    _generate_ranges(keycodes_h_lines, keycodes)
+    _generate_defines(keycodes_h_lines, keycodes)
+    _generate_helpers(keycodes_h_lines, keycodes)
+
+    # Show the results
+    dump_lines(cli.args.output, keycodes_h_lines, cli.args.quiet)
diff --git a/lib/python/qmk/cli/generate/layouts.py b/lib/python/qmk/cli/generate/layouts.py
deleted file mode 100755
index 193633baf6..0000000000
--- a/lib/python/qmk/cli/generate/layouts.py
+++ /dev/null
@@ -1,90 +0,0 @@
-"""Used by the make system to generate layouts.h from info.json.
-"""
-from milc import cli
-
-from qmk.constants import COL_LETTERS, ROW_LETTERS, GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE
-from qmk.decorators import automagic_keyboard, automagic_keymap
-from qmk.info import info_json
-from qmk.keyboard import keyboard_completer, keyboard_folder
-from qmk.path import is_keyboard, normpath
-from qmk.commands import dump_lines
-
-usb_properties = {
-    'vid': 'VENDOR_ID',
-    'pid': 'PRODUCT_ID',
-    'device_ver': 'DEVICE_VER',
-}
-
-
-@cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to')
-@cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages")
-@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='Keyboard to generate config.h for.')
-@cli.subcommand('Used by the make system to generate layouts.h from info.json', hidden=True)
-@automagic_keyboard
-@automagic_keymap
-def generate_layouts(cli):
-    """Generates the layouts.h file.
-    """
-    # Determine our keyboard(s)
-    if not cli.config.generate_layouts.keyboard:
-        cli.log.error('Missing parameter: --keyboard')
-        cli.subcommands['info'].print_help()
-        return False
-
-    if not is_keyboard(cli.config.generate_layouts.keyboard):
-        cli.log.error('Invalid keyboard: "%s"', cli.config.generate_layouts.keyboard)
-        return False
-
-    # Build the info.json file
-    kb_info_json = info_json(cli.config.generate_layouts.keyboard)
-
-    # Build the layouts.h file.
-    layouts_h_lines = [GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE, '#pragma once']
-
-    if 'matrix_size' not in kb_info_json:
-        cli.log.error('%s: Invalid matrix config.', cli.config.generate_layouts.keyboard)
-        return False
-
-    col_num = kb_info_json['matrix_size']['cols']
-    row_num = kb_info_json['matrix_size']['rows']
-
-    for layout_name in kb_info_json['layouts']:
-        if kb_info_json['layouts'][layout_name]['c_macro']:
-            continue
-
-        if 'matrix' not in kb_info_json['layouts'][layout_name]['layout'][0]:
-            cli.log.debug('%s/%s: No matrix data!', cli.config.generate_layouts.keyboard, layout_name)
-            continue
-
-        layout_keys = []
-        layout_matrix = [['KC_NO' for i in range(col_num)] for i in range(row_num)]
-
-        for i, key in enumerate(kb_info_json['layouts'][layout_name]['layout']):
-            row = key['matrix'][0]
-            col = key['matrix'][1]
-            identifier = 'k%s%s' % (ROW_LETTERS[row], COL_LETTERS[col])
-
-            try:
-                layout_matrix[row][col] = identifier
-                layout_keys.append(identifier)
-            except IndexError:
-                key_name = key.get('label', identifier)
-                cli.log.error('Matrix data out of bounds for layout %s at index %s (%s): %s, %s', layout_name, i, key_name, row, col)
-                return False
-
-        layouts_h_lines.append('')
-        layouts_h_lines.append('#define %s(%s) {\\' % (layout_name, ', '.join(layout_keys)))
-
-        rows = ', \\\n'.join(['\t {' + ', '.join(row) + '}' for row in layout_matrix])
-        rows += ' \\'
-        layouts_h_lines.append(rows)
-        layouts_h_lines.append('}')
-
-    for alias, target in kb_info_json.get('layout_aliases', {}).items():
-        layouts_h_lines.append('')
-        layouts_h_lines.append(f'#ifndef {alias}')
-        layouts_h_lines.append(f'#   define {alias} {target}')
-        layouts_h_lines.append('#endif')
-
-    # Show the results
-    dump_lines(cli.args.output, layouts_h_lines, cli.args.quiet)
diff --git a/lib/python/qmk/cli/generate/rules_mk.py b/lib/python/qmk/cli/generate/rules_mk.py
index 9623d00fb5..fc272da6c6 100755
--- a/lib/python/qmk/cli/generate/rules_mk.py
+++ b/lib/python/qmk/cli/generate/rules_mk.py
@@ -1,15 +1,16 @@
 """Used by the make system to generate a rules.mk
 """
 from pathlib import Path
-
 from dotty_dict import dotty
+
+from argcomplete.completers import FilesCompleter
 from milc import cli
 
-from qmk.info import info_json, keymap_json_config
+from qmk.info import info_json
 from qmk.json_schema import json_load
 from qmk.keyboard import keyboard_completer, keyboard_folder
-from qmk.commands import dump_lines
-from qmk.path import normpath
+from qmk.commands import dump_lines, parse_configurator_json
+from qmk.path import normpath, FileType
 from qmk.constants import GPL2_HEADER_SH_LIKE, GENERATED_HEADER_SH_LIKE
 
 
@@ -39,22 +40,29 @@ def process_mapping_rule(kb_info_json, rules_key, info_dict):
     return f'{rules_key} ?= {rules_value}'
 
 
+@cli.argument('filename', nargs='?', arg_only=True, type=FileType('r'), completer=FilesCompleter('.json'), help='A configurator export JSON to be compiled and flashed or a pre-compiled binary firmware file (bin/hex) to be flashed.')
 @cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to')
 @cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages")
 @cli.argument('-e', '--escape', arg_only=True, action='store_true', help="Escape spaces in quiet mode")
-@cli.argument('-kb', '--keyboard', arg_only=True, type=keyboard_folder, completer=keyboard_completer, required=True, help='Keyboard to generate rules.mk for.')
-@cli.argument('-km', '--keymap', arg_only=True, help='Keymap to generate rules.mk for.')
+@cli.argument('-kb', '--keyboard', arg_only=True, type=keyboard_folder, completer=keyboard_completer, help='Keyboard to generate rules.mk for.')
 @cli.subcommand('Used by the make system to generate rules.mk from info.json', hidden=True)
 def generate_rules_mk(cli):
     """Generates a rules.mk file from info.json.
     """
+    converter = None
     # Determine our keyboard/keymap
-    if cli.args.keymap:
-        kb_info_json = dotty(keymap_json_config(cli.args.keyboard, cli.args.keymap))
-    else:
+    if cli.args.filename:
+        user_keymap = parse_configurator_json(cli.args.filename)
+        kb_info_json = dotty(user_keymap.get('config', {}))
+        converter = user_keymap.get('converter', None)
+    elif cli.args.keyboard:
         kb_info_json = dotty(info_json(cli.args.keyboard))
+    else:
+        cli.log.error('You must supply a configurator export or `--keyboard`.')
+        cli.subcommands['generate-rules-mk'].print_help()
+        return False
 
-    info_rules_map = json_load(Path('data/mappings/info_rules.json'))
+    info_rules_map = json_load(Path('data/mappings/info_rules.hjson'))
     rules_mk_lines = [GPL2_HEADER_SH_LIKE, GENERATED_HEADER_SH_LIKE]
 
     # Iterate through the info_rules map to generate basic rules
@@ -82,6 +90,9 @@ def generate_rules_mk(cli):
         else:
             rules_mk_lines.append('CUSTOM_MATRIX ?= yes')
 
+    if converter:
+        rules_mk_lines.append(f'CONVERT_TO ?= {converter}')
+
     # Show the results
     dump_lines(cli.args.output, rules_mk_lines)
 
diff --git a/lib/python/qmk/cli/new/keyboard.py b/lib/python/qmk/cli/new/keyboard.py
index 8d4def1bef..251ad919dd 100644
--- a/lib/python/qmk/cli/new/keyboard.py
+++ b/lib/python/qmk/cli/new/keyboard.py
@@ -210,7 +210,7 @@ def new_keyboard(cli):
 
     # Preprocess any development_board presets
     if mcu in dev_boards:
-        defaults_map = json_load(Path('data/mappings/defaults.json'))
+        defaults_map = json_load(Path('data/mappings/defaults.hjson'))
         board = defaults_map['development_board'][mcu]
 
         mcu = board['processor']
diff --git a/lib/python/qmk/cli/painter/convert_graphics.py b/lib/python/qmk/cli/painter/convert_graphics.py
index bbc30d26ff..2519c49b25 100644
--- a/lib/python/qmk/cli/painter/convert_graphics.py
+++ b/lib/python/qmk/cli/painter/convert_graphics.py
@@ -15,6 +15,7 @@ from PIL import Image
 @cli.argument('-f', '--format', required=True, help='Output format, valid types: %s' % (', '.join(valid_formats.keys())))
 @cli.argument('-r', '--no-rle', arg_only=True, action='store_true', help='Disables the use of RLE when encoding images.')
 @cli.argument('-d', '--no-deltas', arg_only=True, action='store_true', help='Disables the use of delta frames when encoding animations.')
+@cli.argument('-w', '--raw', arg_only=True, action='store_true', help='Writes out the QGF file as raw data instead of c/h combo.')
 @cli.subcommand('Converts an input image to something QMK understands')
 def painter_convert_graphics(cli):
     """Converts an image file to a format that Quantum Painter understands.
@@ -53,6 +54,12 @@ def painter_convert_graphics(cli):
     input_img.save(out_data, "QGF", use_deltas=(not cli.args.no_deltas), use_rle=(not cli.args.no_rle), qmk_format=format, verbose=cli.args.verbose)
     out_bytes = out_data.getvalue()
 
+    if cli.args.raw:
+        raw_file = cli.args.output / (cli.args.input.stem + ".qgf")
+        with open(raw_file, 'wb') as raw:
+            raw.write(out_bytes)
+        return
+
     # Work out the text substitutions for rendering the output data
     subs = {
         'generated_type': 'image',
diff --git a/lib/python/qmk/cli/painter/make_font.py b/lib/python/qmk/cli/painter/make_font.py
index 0762843fd3..c0189920d2 100644
--- a/lib/python/qmk/cli/painter/make_font.py
+++ b/lib/python/qmk/cli/painter/make_font.py
@@ -33,6 +33,7 @@ def painter_make_font_image(cli):
 @cli.argument('-u', '--unicode-glyphs', default='', help='Also generate the specified unicode glyphs.')
 @cli.argument('-f', '--format', required=True, help='Output format, valid types: %s' % (', '.join(valid_formats.keys())))
 @cli.argument('-r', '--no-rle', arg_only=True, action='store_true', help='Disable the use of RLE to minimise converted image size.')
+@cli.argument('-w', '--raw', arg_only=True, action='store_true', help='Writes out the QFF file as raw data instead of c/h combo.')
 @cli.subcommand('Converts an input font image to something QMK firmware understands')
 def painter_convert_font_image(cli):
     # Work out the format
@@ -53,6 +54,13 @@ def painter_convert_font_image(cli):
     # Render out the data
     out_data = BytesIO()
     font.save_to_qff(format, (False if cli.args.no_rle else True), out_data)
+    out_bytes = out_data.getvalue()
+
+    if cli.args.raw:
+        raw_file = cli.args.output / (cli.args.input.stem + ".qff")
+        with open(raw_file, 'wb') as raw:
+            raw.write(out_bytes)
+        return
 
     # Work out the text substitutions for rendering the output data
     subs = {
@@ -62,8 +70,8 @@ def painter_convert_font_image(cli):
         'year': datetime.date.today().strftime("%Y"),
         'input_file': cli.args.input.name,
         'sane_name': re.sub(r"[^a-zA-Z0-9]", "_", cli.args.input.stem),
-        'byte_count': out_data.getbuffer().nbytes,
-        'bytes_lines': render_bytes(out_data.getbuffer().tobytes()),
+        'byte_count': len(out_bytes),
+        'bytes_lines': render_bytes(out_bytes),
         'format': cli.args.format,
     }
 
diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py
index 9c0a5dce56..5561a354c5 100644
--- a/lib/python/qmk/commands.py
+++ b/lib/python/qmk/commands.py
@@ -2,13 +2,13 @@
 """
 import os
 import sys
+import json
 import shutil
 from pathlib import Path
 
 from milc import cli
 import jsonschema
 
-import qmk.keymap
 from qmk.constants import KEYBOARD_OUTPUT_PREFIX
 from qmk.json_schema import json_load, validate
 
@@ -107,7 +107,7 @@ def get_make_parallel_args(parallel=1):
     return parallel_args
 
 
-def compile_configurator_json(user_keymap, bootloader=None, parallel=1, **env_vars):
+def compile_configurator_json(user_keymap, bootloader=None, parallel=1, clean=False, **env_vars):
     """Convert a configurator export JSON file into a C file and then compile it.
 
     Args:
@@ -129,17 +129,32 @@ def compile_configurator_json(user_keymap, bootloader=None, parallel=1, **env_va
     # e.g.: qmk compile - < keyboards/clueboard/california/keymaps/default/keymap.json
     user_keymap["keymap"] = user_keymap.get("keymap", "default_json")
 
-    # Write the keymap.c file
     keyboard_filesafe = user_keymap['keyboard'].replace('/', '_')
     target = f'{keyboard_filesafe}_{user_keymap["keymap"]}'
     keyboard_output = Path(f'{KEYBOARD_OUTPUT_PREFIX}{keyboard_filesafe}')
     keymap_output = Path(f'{keyboard_output}_{user_keymap["keymap"]}')
-    c_text = qmk.keymap.generate_c(user_keymap)
     keymap_dir = keymap_output / 'src'
-    keymap_c = keymap_dir / 'keymap.c'
+    keymap_json = keymap_dir / 'keymap.json'
 
+    if clean:
+        if keyboard_output.exists():
+            shutil.rmtree(keyboard_output)
+        if keymap_output.exists():
+            shutil.rmtree(keymap_output)
+
+    # begin with making the deepest folder in the tree
     keymap_dir.mkdir(exist_ok=True, parents=True)
-    keymap_c.write_text(c_text)
+
+    # Compare minified to ensure consistent comparison
+    new_content = json.dumps(user_keymap, separators=(',', ':'))
+    if keymap_json.exists():
+        old_content = json.dumps(json.loads(keymap_json.read_text(encoding='utf-8')), separators=(',', ':'))
+        if old_content == new_content:
+            new_content = None
+
+    # Write the keymap.json file if different
+    if new_content:
+        keymap_json.write_text(new_content, encoding='utf-8')
 
     # Return a command that can be run to make the keymap and flash if given
     verbose = 'true' if cli.config.general.verbose else 'false'
@@ -175,7 +190,7 @@ def compile_configurator_json(user_keymap, bootloader=None, parallel=1, **env_va
         f'MAIN_KEYMAP_PATH_3={keymap_output}',
         f'MAIN_KEYMAP_PATH_4={keymap_output}',
         f'MAIN_KEYMAP_PATH_5={keymap_output}',
-        f'KEYMAP_C={keymap_c}',
+        f'KEYMAP_JSON={keymap_json}',
         f'KEYMAP_PATH={keymap_dir}',
         f'VERBOSE={verbose}',
         f'COLOR={color}',
@@ -199,7 +214,7 @@ def parse_configurator_json(configurator_file):
         exit(1)
 
     orig_keyboard = user_keymap['keyboard']
-    aliases = json_load(Path('data/mappings/keyboard_aliases.json'))
+    aliases = json_load(Path('data/mappings/keyboard_aliases.hjson'))
 
     if orig_keyboard in aliases:
         if 'target' in aliases[orig_keyboard]:
@@ -211,6 +226,19 @@ def parse_configurator_json(configurator_file):
     return user_keymap
 
 
+def build_environment(args):
+    """Common processing for cli.args.env
+    """
+    envs = {}
+    for env in args:
+        if '=' in env:
+            key, value = env.split('=', 1)
+            envs[key] = value
+        else:
+            cli.log.warning('Invalid environment variable: %s', env)
+    return envs
+
+
 def in_virtualenv():
     """Check if running inside a virtualenv.
     Based on https://stackoverflow.com/a/1883251
diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py
index 98505c1c8e..1692d43c9c 100644
--- a/lib/python/qmk/info.py
+++ b/lib/python/qmk/info.py
@@ -437,19 +437,6 @@ def _extract_matrix_info(info_data, config_c):
     return info_data
 
 
-# TODO: kill off usb.device_ver in favor of usb.device_version
-def _extract_device_version(info_data):
-    if info_data.get('usb'):
-        if info_data['usb'].get('device_version') and not info_data['usb'].get('device_ver'):
-            (major, minor, revision) = info_data['usb']['device_version'].split('.', 3)
-            info_data['usb']['device_ver'] = f'0x{major.zfill(2)}{minor}{revision}'
-        if not info_data['usb'].get('device_version') and info_data['usb'].get('device_ver'):
-            major = int(info_data['usb']['device_ver'][2:4])
-            minor = int(info_data['usb']['device_ver'][4])
-            revision = int(info_data['usb']['device_ver'][5])
-            info_data['usb']['device_version'] = f'{major}.{minor}.{revision}'
-
-
 def _config_to_json(key_type, config_value):
     """Convert config value using spec
     """
@@ -464,7 +451,7 @@ def _config_to_json(key_type, config_value):
         if array_type == 'int':
             return list(map(int, config_value.split(',')))
         else:
-            return config_value.split(',')
+            return list(map(str.strip, config_value.split(',')))
 
     elif key_type == 'bool':
         return config_value in true_values
@@ -479,7 +466,7 @@ def _config_to_json(key_type, config_value):
         return int(config_value)
 
     elif key_type == 'str':
-        return config_value.strip('"')
+        return config_value.strip('"').replace('\\"', '"').replace('\\\\', '\\')
 
     elif key_type == 'bcd_version':
         major = int(config_value[2:4])
@@ -496,7 +483,7 @@ def _extract_config_h(info_data, config_c):
     """
     # Pull in data from the json map
     dotty_info = dotty(info_data)
-    info_config_map = json_load(Path('data/mappings/info_config.json'))
+    info_config_map = json_load(Path('data/mappings/info_config.hjson'))
 
     for config_key, info_dict in info_config_map.items():
         info_key = info_dict['info_key']
@@ -535,7 +522,6 @@ def _extract_config_h(info_data, config_c):
     _extract_split_right_pins(info_data, config_c)
     _extract_encoders(info_data, config_c)
     _extract_split_encoders(info_data, config_c)
-    _extract_device_version(info_data)
 
     return info_data
 
@@ -543,7 +529,7 @@ def _extract_config_h(info_data, config_c):
 def _process_defaults(info_data):
     """Process any additional defaults based on currently discovered information
     """
-    defaults_map = json_load(Path('data/mappings/defaults.json'))
+    defaults_map = json_load(Path('data/mappings/defaults.hjson'))
     for default_type in defaults_map.keys():
         thing_map = defaults_map[default_type]
         if default_type in info_data:
@@ -569,7 +555,7 @@ def _extract_rules_mk(info_data, rules):
 
     # Pull in data from the json map
     dotty_info = dotty(info_data)
-    info_rules_map = json_load(Path('data/mappings/info_rules.json'))
+    info_rules_map = json_load(Path('data/mappings/info_rules.hjson'))
 
     for rules_key, info_dict in info_rules_map.items():
         info_key = info_dict['info_key']
@@ -627,20 +613,24 @@ def _extract_led_config(info_data, keyboard):
     cols = info_data['matrix_size']['cols']
     rows = info_data['matrix_size']['rows']
 
-    # Assume what feature owns g_led_config
-    feature = "rgb_matrix"
-    if info_data.get("features", {}).get("led_matrix", False):
+    # Determine what feature owns g_led_config
+    features = info_data.get("features", {})
+    feature = None
+    if features.get("rgb_matrix", False):
+        feature = "rgb_matrix"
+    elif features.get("led_matrix", False):
         feature = "led_matrix"
 
-    # Process
-    for file in find_keyboard_c(keyboard):
-        try:
-            ret = find_led_config(file, cols, rows)
-            if ret:
-                info_data[feature] = info_data.get(feature, {})
-                info_data[feature]["layout"] = ret
-        except Exception as e:
-            _log_warning(info_data, f'led_config: {file.name}: {e}')
+    if feature:
+        # Process
+        for file in find_keyboard_c(keyboard):
+            try:
+                ret = find_led_config(file, cols, rows)
+                if ret:
+                    info_data[feature] = info_data.get(feature, {})
+                    info_data[feature]["layout"] = ret
+            except Exception as e:
+                _log_warning(info_data, f'led_config: {file.name}: {e}')
 
     return info_data
 
@@ -755,9 +745,6 @@ def arm_processor_rules(info_data, rules):
     info_data['processor_type'] = 'arm'
     info_data['protocol'] = 'ChibiOS'
 
-    if 'bootloader' not in info_data:
-        info_data['bootloader'] = 'unknown'
-
     if 'STM32' in info_data['processor']:
         info_data['platform'] = 'STM32'
     elif 'MCU_SERIES' in rules:
@@ -775,9 +762,6 @@ def avr_processor_rules(info_data, rules):
     info_data['platform'] = rules['ARCH'] if 'ARCH' in rules else 'unknown'
     info_data['protocol'] = 'V-USB' if info_data['processor'] in VUSB_PROCESSORS else 'LUFA'
 
-    if 'bootloader' not in info_data:
-        info_data['bootloader'] = 'atmel-dfu'
-
     # FIXME(fauxpark/anyone): Eventually we should detect the protocol by looking at PROTOCOL inherited from mcu_selection.mk:
     # info_data['protocol'] = 'V-USB' if rules.get('PROTOCOL') == 'VUSB' else 'LUFA'
 
diff --git a/lib/python/qmk/json_schema.py b/lib/python/qmk/json_schema.py
index 01175146b5..934e2f841f 100644
--- a/lib/python/qmk/json_schema.py
+++ b/lib/python/qmk/json_schema.py
@@ -10,7 +10,18 @@ import jsonschema
 from milc import cli
 
 
-def json_load(json_file):
+def _dict_raise_on_duplicates(ordered_pairs):
+    """Reject duplicate keys."""
+    d = {}
+    for k, v in ordered_pairs:
+        if k in d:
+            raise ValueError("duplicate key: %r" % (k,))
+        else:
+            d[k] = v
+    return d
+
+
+def json_load(json_file, strict=True):
     """Load a json file from disk.
 
     Note: file must be a Path object.
@@ -20,7 +31,7 @@ def json_load(json_file):
         # Not necessary if the data is provided via stdin
         if isinstance(json_file, Path):
             json_file = json_file.open(encoding='utf-8')
-        return hjson.load(json_file)
+        return hjson.load(json_file, object_pairs_hook=_dict_raise_on_duplicates if strict else None)
 
     except (json.decoder.JSONDecodeError, hjson.HjsonDecodeError) as e:
         cli.log.error('Invalid JSON encountered attempting to load {fg_cyan}%s{fg_reset}:\n\t{fg_red}%s', json_file, e)
diff --git a/lib/python/qmk/keyboard.py b/lib/python/qmk/keyboard.py
index 686d4fc403..6ddbba8fa5 100644
--- a/lib/python/qmk/keyboard.py
+++ b/lib/python/qmk/keyboard.py
@@ -69,7 +69,7 @@ def keyboard_folder(keyboard):
 
     This checks aliases and DEFAULT_FOLDER to resolve the actual path for a keyboard.
     """
-    aliases = json_load(Path('data/mappings/keyboard_aliases.json'))
+    aliases = json_load(Path('data/mappings/keyboard_aliases.hjson'))
 
     if keyboard in aliases:
         keyboard = aliases[keyboard].get('target', keyboard)
diff --git a/lib/python/qmk/keycodes.py b/lib/python/qmk/keycodes.py
new file mode 100644
index 0000000000..cf1ee0767a
--- /dev/null
+++ b/lib/python/qmk/keycodes.py
@@ -0,0 +1,57 @@
+from pathlib import Path
+
+from qmk.json_schema import deep_update, json_load, validate
+
+CONSTANTS_PATH = Path('data/constants/keycodes/')
+
+
+def _validate(spec):
+    # first throw it to the jsonschema
+    validate(spec, 'qmk.keycodes.v1')
+
+    # no duplicate keycodes
+    keycodes = []
+    for value in spec['keycodes'].values():
+        keycodes.append(value['key'])
+        keycodes.extend(value.get('aliases', []))
+    duplicates = set([x for x in keycodes if keycodes.count(x) > 1])
+    if duplicates:
+        raise ValueError(f'Keycode spec contains duplicate keycodes! ({",".join(duplicates)})')
+
+
+def load_spec(version):
+    """Build keycode data from the requested spec file
+    """
+    if version == 'latest':
+        version = list_versions()[0]
+
+    file = CONSTANTS_PATH / f'keycodes_{version}.hjson'
+    if not file.exists():
+        raise ValueError(f'Requested keycode spec ({version}) is invalid!')
+
+    # Load base
+    spec = json_load(file)
+
+    # Merge in fragments
+    fragments = CONSTANTS_PATH.glob(f'keycodes_{version}_*.hjson')
+    for file in fragments:
+        deep_update(spec, json_load(file))
+
+    # Sort?
+    spec['keycodes'] = dict(sorted(spec['keycodes'].items()))
+
+    # Validate?
+    _validate(spec)
+
+    return spec
+
+
+def list_versions():
+    """Return available versions - sorted newest first
+    """
+    ret = []
+    for file in CONSTANTS_PATH.glob('keycodes_[0-9].[0-9].[0-9].hjson'):
+        ret.append(file.stem.split('_')[1])
+
+    ret.sort(reverse=True)
+    return ret
diff --git a/lib/python/qmk/keymap.py b/lib/python/qmk/keymap.py
index f317f4d11e..315af35b73 100644
--- a/lib/python/qmk/keymap.py
+++ b/lib/python/qmk/keymap.py
@@ -266,7 +266,7 @@ def generate_c(keymap_json):
 
             new_macro = "".join(macro)
             new_macro = new_macro.replace('""', '')
-            macro_txt.append(f'            case MACRO_{i}:')
+            macro_txt.append(f'            case QK_MACRO_{i}:')
             macro_txt.append(f'                SEND_STRING({new_macro});')
             macro_txt.append('                return false;')
 
@@ -412,7 +412,7 @@ def list_keymaps(keyboard, c=True, json=True, additional_files=None, fullpath=Fa
     rules = rules_mk(keyboard)
     names = set()
 
-    if rules:
+    if rules is not None:
         keyboards_dir = Path('keyboards')
         kb_path = keyboards_dir / keyboard
 
diff --git a/lib/python/qmk/tests/test_cli_commands.py b/lib/python/qmk/tests/test_cli_commands.py
index fde8b079a3..e598b281a6 100644
--- a/lib/python/qmk/tests/test_cli_commands.py
+++ b/lib/python/qmk/tests/test_cli_commands.py
@@ -150,8 +150,8 @@ def test_json2c():
 def test_json2c_macros():
     result = check_subcommand("json2c", 'keyboards/handwired/pytest/macro/keymaps/default/keymap.json')
     check_returncode(result)
-    assert 'LAYOUT_ortho_1x1(MACRO_0)' in result.stdout
-    assert 'case MACRO_0:' in result.stdout
+    assert 'LAYOUT_ortho_1x1(QK_MACRO_0)' in result.stdout
+    assert 'case QK_MACRO_0:' in result.stdout
     assert 'SEND_STRING("Hello, World!"SS_TAP(X_ENTER));' in result.stdout
 
 
@@ -263,16 +263,16 @@ def test_generate_rgb_breathe_table():
 def test_generate_config_h():
     result = check_subcommand('generate-config-h', '-kb', 'handwired/pytest/basic')
     check_returncode(result)
-    assert '#   define DEVICE_VER 0x0001' in result.stdout
-    assert '#   define DIODE_DIRECTION COL2ROW' in result.stdout
-    assert '#   define MANUFACTURER none' in result.stdout
-    assert '#   define PRODUCT pytest' in result.stdout
-    assert '#   define PRODUCT_ID 0x6465' in result.stdout
-    assert '#   define VENDOR_ID 0xFEED' in result.stdout
-    assert '#   define MATRIX_COLS 1' in result.stdout
-    assert '#   define MATRIX_COL_PINS { F4 }' in result.stdout
-    assert '#   define MATRIX_ROWS 1' in result.stdout
-    assert '#   define MATRIX_ROW_PINS { F5 }' in result.stdout
+    assert '#    define DEVICE_VER 0x0001' in result.stdout
+    assert '#    define DIODE_DIRECTION COL2ROW' in result.stdout
+    assert '#    define MANUFACTURER "none"' in result.stdout
+    assert '#    define PRODUCT "pytest"' in result.stdout
+    assert '#    define PRODUCT_ID 0x6465' in result.stdout
+    assert '#    define VENDOR_ID 0xFEED' in result.stdout
+    assert '#    define MATRIX_COLS 1' in result.stdout
+    assert '#    define MATRIX_COL_PINS { F4 }' in result.stdout
+    assert '#    define MATRIX_ROWS 1' in result.stdout
+    assert '#    define MATRIX_ROW_PINS { F5 }' in result.stdout
 
 
 def test_generate_rules_mk():
@@ -288,12 +288,6 @@ def test_generate_version_h():
     assert '#define QMK_VERSION' in result.stdout
 
 
-def test_generate_layouts():
-    result = check_subcommand('generate-layouts', '-kb', 'handwired/pytest/basic')
-    check_returncode(result)
-    assert '#define LAYOUT_custom(k0A) {' in result.stdout
-
-
 def test_format_json_keyboard():
     result = check_subcommand('format-json', '--format', 'keyboard', 'lib/python/qmk/tests/minimal_info.json')
     check_returncode(result)