diff options
Diffstat (limited to 'lib/python')
| -rw-r--r-- | lib/python/qmk/cli/__init__.py | 3 | ||||
| -rwxr-xr-x | lib/python/qmk/cli/compile.py | 57 | ||||
| -rw-r--r-- | lib/python/qmk/cli/flash.py | 99 | ||||
| -rwxr-xr-x | lib/python/qmk/cli/generate/api.py | 35 | ||||
| -rw-r--r-- | lib/python/qmk/cli/generate/autocorrect_data.py | 289 | ||||
| -rwxr-xr-x | lib/python/qmk/cli/generate/config_h.py | 156 | ||||
| -rwxr-xr-x | lib/python/qmk/cli/generate/keyboard_h.py | 69 | ||||
| -rw-r--r-- | lib/python/qmk/cli/generate/keycodes.py | 88 | ||||
| -rwxr-xr-x | lib/python/qmk/cli/generate/layouts.py | 90 | ||||
| -rwxr-xr-x | lib/python/qmk/cli/generate/rules_mk.py | 31 | ||||
| -rw-r--r-- | lib/python/qmk/cli/new/keyboard.py | 2 | ||||
| -rw-r--r-- | lib/python/qmk/commands.py | 44 | ||||
| -rw-r--r-- | lib/python/qmk/info.py | 56 | ||||
| -rw-r--r-- | lib/python/qmk/keyboard.py | 2 | ||||
| -rw-r--r-- | lib/python/qmk/keycodes.py | 57 | ||||
| -rw-r--r-- | lib/python/qmk/keymap.py | 4 | ||||
| -rw-r--r-- | lib/python/qmk/tests/test_cli_commands.py | 30 |
17 files changed, 743 insertions, 369 deletions
diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py index 98e212c47b..9190af4e50 100644 --- a/lib/python/qmk/cli/__init__.py +++ b/lib/python/qmk/cli/__init__.py @@ -47,6 +47,7 @@ subcommands = [ 'qmk.cli.format.python', 'qmk.cli.format.text', 'qmk.cli.generate.api', + 'qmk.cli.generate.autocorrect_data', 'qmk.cli.generate.compilation_database', 'qmk.cli.generate.config_h', 'qmk.cli.generate.develop_pr_list', @@ -55,7 +56,7 @@ subcommands = [ 'qmk.cli.generate.info_json', 'qmk.cli.generate.keyboard_c', 'qmk.cli.generate.keyboard_h', - 'qmk.cli.generate.layouts', + 'qmk.cli.generate.keycodes', 'qmk.cli.generate.rgb_breathe_table', 'qmk.cli.generate.rules_mk', 'qmk.cli.generate.version_h', diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py index 95118e6687..9e7629906f 100755 --- a/lib/python/qmk/cli/compile.py +++ b/lib/python/qmk/cli/compile.py @@ -2,14 +2,13 @@ You can compile a keymap already in the repo or using a QMK Configurator export. """ -from subprocess import DEVNULL - from argcomplete.completers import FilesCompleter + from milc import cli import qmk.path from qmk.decorators import automagic_keyboard, automagic_keymap -from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json +from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json, build_environment from qmk.keyboard import keyboard_completer, keyboard_folder from qmk.keymap import keymap_completer @@ -31,48 +30,32 @@ def compile(cli): If a keyboard and keymap are provided this command will build a firmware based on that. """ - if cli.args.clean and not cli.args.filename and not cli.args.dry_run: - if cli.config.compile.keyboard and cli.config.compile.keymap: - command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, 'clean') - cli.run(command, capture_output=False, stdin=DEVNULL) - # Build the environment vars - envs = {} - for env in cli.args.env: - if '=' in env: - key, value = env.split('=', 1) - envs[key] = value - else: - cli.log.warning('Invalid environment variable: %s', env) + envs = build_environment(cli.args.env) # Determine the compile command - command = None + commands = [] if cli.args.filename: # If a configurator JSON was provided generate a keymap and compile it user_keymap = parse_configurator_json(cli.args.filename) - command = compile_configurator_json(user_keymap, parallel=cli.config.compile.parallel, **envs) + commands = [compile_configurator_json(user_keymap, parallel=cli.config.compile.parallel, clean=cli.args.clean, **envs)] - else: - if cli.config.compile.keyboard and cli.config.compile.keymap: - # Generate the make command for a specific keyboard/keymap. - command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, parallel=cli.config.compile.parallel, **envs) + elif cli.config.compile.keyboard and cli.config.compile.keymap: + # Generate the make command for a specific keyboard/keymap. + if cli.args.clean: + commands.append(create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, 'clean', **envs)) + commands.append(create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, parallel=cli.config.compile.parallel, **envs)) - elif not cli.config.compile.keyboard: - cli.log.error('Could not determine keyboard!') - elif not cli.config.compile.keymap: - cli.log.error('Could not determine keymap!') - - # Compile the firmware, if we're able to - if command: - cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command)) - if not cli.args.dry_run: - cli.echo('\n') - # FIXME(skullydazed/anyone): Remove text=False once milc 1.0.11 has had enough time to be installed everywhere. - compile = cli.run(command, capture_output=False, text=False) - return compile.returncode - - else: + if not commands: cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.') - cli.echo('usage: qmk compile [-h] [-b] [-kb KEYBOARD] [-km KEYMAP] [filename]') + cli.print_help() return False + + cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(commands[-1])) + if not cli.args.dry_run: + cli.echo('\n') + for command in commands: + ret = cli.run(command, capture_output=False) + if ret.returncode: + return ret.returncode diff --git a/lib/python/qmk/cli/flash.py b/lib/python/qmk/cli/flash.py index c39f4b36d4..40bfbdab56 100644 --- a/lib/python/qmk/cli/flash.py +++ b/lib/python/qmk/cli/flash.py @@ -3,15 +3,13 @@ You can compile a keymap already in the repo or using a QMK Configurator export. A bootloader must be specified. """ -from subprocess import DEVNULL -import sys - from argcomplete.completers import FilesCompleter + from milc import cli import qmk.path from qmk.decorators import automagic_keyboard, automagic_keymap -from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json +from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json, build_environment from qmk.keyboard import keyboard_completer, keyboard_folder from qmk.flashers import flasher @@ -75,59 +73,40 @@ def flash(cli): return False except KeyboardInterrupt: cli.log.info('Ctrl-C was pressed, exiting...') - sys.exit(0) - - else: - if cli.args.clean and not cli.args.filename and not cli.args.dry_run: - if cli.config.flash.keyboard and cli.config.flash.keymap: - command = create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, 'clean') - cli.run(command, capture_output=False, stdin=DEVNULL) - - # Build the environment vars - envs = {} - for env in cli.args.env: - if '=' in env: - key, value = env.split('=', 1) - envs[key] = value - else: - cli.log.warning('Invalid environment variable: %s', env) - - # Determine the compile command - command = '' - - if cli.args.bootloaders: - # Provide usage and list bootloaders - cli.echo('usage: qmk flash [-h] [-b] [-n] [-kb KEYBOARD] [-km KEYMAP] [-bl BOOTLOADER] [filename]') - print_bootloader_help() - return False - - if cli.args.filename: - # Handle compiling a configurator JSON - user_keymap = parse_configurator_json(cli.args.filename) - keymap_path = qmk.path.keymap(user_keymap['keyboard']) - command = compile_configurator_json(user_keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, **envs) - - cli.log.info('Wrote keymap to {fg_cyan}%s/%s/keymap.c', keymap_path, user_keymap['keymap']) - - else: - if cli.config.flash.keyboard and cli.config.flash.keymap: - # Generate the make command for a specific keyboard/keymap. - command = create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, **envs) - - elif not cli.config.flash.keyboard: - cli.log.error('Could not determine keyboard!') - elif not cli.config.flash.keymap: - cli.log.error('Could not determine keymap!') - - # Compile the firmware, if we're able to - if command: - cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command)) - if not cli.args.dry_run: - cli.echo('\n') - compile = cli.run(command, capture_output=False, stdin=DEVNULL) - return compile.returncode - - else: - cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.') - cli.echo('usage: qmk flash [-h] [-b] [-n] [-kb KEYBOARD] [-km KEYMAP] [-bl BOOTLOADER] [filename]') - return False + return True + + if cli.args.bootloaders: + # Provide usage and list bootloaders + cli.print_help() + print_bootloader_help() + return False + + # Build the environment vars + envs = build_environment(cli.args.env) + + # Determine the compile command + commands = [] + + if cli.args.filename: + # If a configurator JSON was provided generate a keymap and compile it + user_keymap = parse_configurator_json(cli.args.filename) + commands = [compile_configurator_json(user_keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, clean=cli.args.clean, **envs)] + + elif cli.config.flash.keyboard and cli.config.flash.keymap: + # Generate the make command for a specific keyboard/keymap. + if cli.args.clean: + commands.append(create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, 'clean', **envs)) + commands.append(create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, **envs)) + + if not commands: + cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.') + cli.print_help() + return False + + cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(commands[-1])) + if not cli.args.dry_run: + cli.echo('\n') + for command in commands: + ret = cli.run(command, capture_output=False) + if ret.returncode: + return ret.returncode diff --git a/lib/python/qmk/cli/generate/api.py b/lib/python/qmk/cli/generate/api.py index 8d8ca3cd41..a98a12b628 100755 --- a/lib/python/qmk/cli/generate/api.py +++ b/lib/python/qmk/cli/generate/api.py @@ -11,12 +11,40 @@ from qmk.info import info_json from qmk.json_encoders import InfoJSONEncoder from qmk.json_schema import json_load from qmk.keyboard import find_readme, list_keyboards +from qmk.keycodes import load_spec, list_versions DATA_PATH = Path('data') TEMPLATE_PATH = DATA_PATH / 'templates/api/' BUILD_API_PATH = Path('.build/api_data/') +def _resolve_keycode_specs(output_folder): + """To make it easier for consumers, publish pre-merged spec files + """ + for version in list_versions(): + overall = load_spec(version) + + output_file = output_folder / f'constants/keycodes_{version}.json' + output_file.write_text(json.dumps(overall, indent=4), encoding='utf-8') + + # Purge files consumed by 'load_spec' + shutil.rmtree(output_folder / 'constants/keycodes/') + + +def _filtered_copy(src, dst): + src = Path(src) + dst = Path(dst) + + if dst.suffix == '.hjson': + data = json_load(src) + + dst = dst.with_suffix('.json') + dst.write_text(json.dumps(data, indent=4), encoding='utf-8') + return dst + + return shutil.copy2(src, dst) + + def _filtered_keyboard_list(): """Perform basic filtering of list_keyboards """ @@ -47,7 +75,7 @@ def generate_api(cli): shutil.rmtree(BUILD_API_PATH) shutil.copytree(TEMPLATE_PATH, BUILD_API_PATH) - shutil.copytree(DATA_PATH, v1_dir) + shutil.copytree(DATA_PATH, v1_dir, copy_function=_filtered_copy) # Filter down when required keyboard_list = _filtered_keyboard_list() @@ -87,7 +115,7 @@ def generate_api(cli): # Generate data for the global files keyboard_list = sorted(kb_all) - keyboard_aliases = json_load(Path('data/mappings/keyboard_aliases.json')) + keyboard_aliases = json_load(Path('data/mappings/keyboard_aliases.hjson')) keyboard_metadata = { 'last_updated': current_datetime(), 'keyboards': keyboard_list, @@ -95,6 +123,9 @@ def generate_api(cli): 'usb': usb_list, } + # Feature specific handling + _resolve_keycode_specs(v1_dir) + # Write the global JSON files keyboard_all_json = json.dumps({'last_updated': current_datetime(), 'keyboards': kb_all}, cls=InfoJSONEncoder) usb_json = json.dumps({'last_updated': current_datetime(), 'usb': usb_list}, cls=InfoJSONEncoder) diff --git a/lib/python/qmk/cli/generate/autocorrect_data.py b/lib/python/qmk/cli/generate/autocorrect_data.py new file mode 100644 index 0000000000..00ab6180ab --- /dev/null +++ b/lib/python/qmk/cli/generate/autocorrect_data.py @@ -0,0 +1,289 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Python program to make autocorrect_data.h. +This program reads from a prepared dictionary file and generates a C source file +"autocorrect_data.h" with a serialized trie embedded as an array. Run this +program and pass it as the first argument like: +$ qmk generate-autocorrect-data autocorrect_dict.txt +Each line of the dict file defines one typo and its correction with the syntax +"typo -> correction". Blank lines or lines starting with '#' are ignored. +Example: + :thier -> their + fitler -> filter + lenght -> length + ouput -> output + widht -> width +For full documentation, see QMK Docs +""" + +import sys +import textwrap +from typing import Any, Dict, Iterator, List, Tuple + +from milc import cli + +import qmk.path +from qmk.keyboard import keyboard_completer, keyboard_folder +from qmk.keymap import keymap_completer, locate_keymap + +KC_A = 4 +KC_SPC = 0x2c +KC_QUOT = 0x34 + +TYPO_CHARS = dict([ + ("'", KC_QUOT), + (':', KC_SPC), # "Word break" character. +] + [(chr(c), c + KC_A - ord('a')) for c in range(ord('a'), + ord('z') + 1)]) # Characters a-z. + + +def parse_file(file_name: str) -> List[Tuple[str, str]]: + """Parses autocorrections dictionary file. + Each line of the file defines one typo and its correction with the syntax + "typo -> correction". Blank lines or lines starting with '#' are ignored. The + function validates that typos only have characters a-z and that typos are not + substrings of other typos, otherwise the longer typo would never trigger. + Args: + file_name: String, path of the autocorrections dictionary. + Returns: + List of (typo, correction) tuples. + """ + + try: + from english_words import english_words_lower_alpha_set as correct_words + except ImportError: + cli.echo('Autocorrection will falsely trigger when a typo is a substring of a correctly spelled word.') + cli.echo('To check for this, install the english_words package and rerun this script:') + cli.echo(' {fg_cyan}python3 -m pip install english_words') + # Use a minimal word list as a fallback. + correct_words = ('information', 'available', 'international', 'language', 'loosest', 'reference', 'wealthier', 'entertainment', 'association', 'provides', 'technology', 'statehood') + + autocorrections = [] + typos = set() + for line_number, typo, correction in parse_file_lines(file_name): + if typo in typos: + cli.log.warning('{fg_red}Error:%d:{fg_reset} Ignoring duplicate typo: "{fg_cyan}%s{fg_reset}"', line_number, typo) + continue + + # Check that `typo` is valid. + if not (all([c in TYPO_CHARS for c in typo])): + cli.log.error('{fg_red}Error:%d:{fg_reset} Typo "{fg_cyan}%s{fg_reset}" has characters other than a-z, \' and :.', line_number, typo) + sys.exit(1) + for other_typo in typos: + if typo in other_typo or other_typo in typo: + cli.log.error('{fg_red}Error:%d:{fg_reset} Typos may not be substrings of one another, otherwise the longer typo would never trigger: "{fg_cyan}%s{fg_reset}" vs. "{fg_cyan}%s{fg_reset}".', line_number, typo, other_typo) + sys.exit(1) + if len(typo) < 5: + cli.log.warning('{fg_yellow}Warning:%d:{fg_reset} It is suggested that typos are at least 5 characters long to avoid false triggers: "{fg_cyan}%s{fg_reset}"', line_number, typo) + if len(typo) > 127: + cli.log.error('{fg_red}Error:%d:{fg_reset} Typo exceeds 127 chars: "{fg_cyan}%s{fg_reset}"', line_number, typo) + sys.exit(1) + + check_typo_against_dictionary(typo, line_number, correct_words) + + autocorrections.append((typo, correction)) + typos.add(typo) + + return autocorrections + + +def make_trie(autocorrections: List[Tuple[str, str]]) -> Dict[str, Any]: + """Makes a trie from the the typos, writing in reverse. + Args: + autocorrections: List of (typo, correction) tuples. + Returns: + Dict of dict, representing the trie. + """ + trie = {} + for typo, correction in autocorrections: + node = trie + for letter in typo[::-1]: + node = node.setdefault(letter, {}) + node['LEAF'] = (typo, correction) + + return trie + + +def parse_file_lines(file_name: str) -> Iterator[Tuple[int, str, str]]: + """Parses lines read from `file_name` into typo-correction pairs.""" + + line_number = 0 + for line in open(file_name, 'rt'): + line_number += 1 + line = line.strip() + if line and line[0] != '#': + # Parse syntax "typo -> correction", using strip to ignore indenting. + tokens = [token.strip() for token in line.split('->', 1)] + if len(tokens) != 2 or not tokens[0]: + print(f'Error:{line_number}: Invalid syntax: "{line}"') + sys.exit(1) + + typo, correction = tokens + typo = typo.lower() # Force typos to lowercase. + typo = typo.replace(' ', ':') + + yield line_number, typo, correction + + +def check_typo_against_dictionary(typo: str, line_number: int, correct_words) -> None: + """Checks `typo` against English dictionary words.""" + + if typo.startswith(':') and typo.endswith(':'): + if typo[1:-1] in correct_words: + cli.log.warning('{fg_yellow}Warning:%d:{fg_reset} Typo "{fg_cyan}%s{fg_reset}" is a correctly spelled dictionary word.', line_number, typo) + elif typo.startswith(':') and not typo.endswith(':'): + for word in correct_words: + if word.startswith(typo[1:]): + cli.log.warning('{fg_yellow}Warning:%d: {fg_reset}Typo "{fg_cyan}%s{fg_reset}" would falsely trigger on correctly spelled word "{fg_cyan}%s{fg_reset}".', line_number, typo, word) + elif not typo.startswith(':') and typo.endswith(':'): + for word in correct_words: + if word.endswith(typo[:-1]): + cli.log.warning('{fg_yellow}Warning:%d:{fg_reset} Typo "{fg_cyan}%s{fg_reset}" would falsely trigger on correctly spelled word "{fg_cyan}%s{fg_reset}".', line_number, typo, word) + elif not typo.startswith(':') and not typo.endswith(':'): + for word in correct_words: + if typo in word: + cli.log.warning('{fg_yellow}Warning:%d:{fg_reset} Typo "{fg_cyan}%s{fg_reset}" would falsely trigger on correctly spelled word "{fg_cyan}%s{fg_reset}".', line_number, typo, word) + + +def serialize_trie(autocorrections: List[Tuple[str, str]], trie: Dict[str, Any]) -> List[int]: + """Serializes trie and correction data in a form readable by the C code. + Args: + autocorrections: List of (typo, correction) tuples. + trie: Dict of dicts. + Returns: + List of ints in the range 0-255. + """ + table = [] + + # Traverse trie in depth first order. + def traverse(trie_node): + if 'LEAF' in trie_node: # Handle a leaf trie node. + typo, correction = trie_node['LEAF'] + word_boundary_ending = typo[-1] == ':' + typo = typo.strip(':') + i = 0 # Make the autocorrection data for this entry and serialize it. + while i < min(len(typo), len(correction)) and typo[i] == correction[i]: + i += 1 + backspaces = len(typo) - i - 1 + word_boundary_ending + assert 0 <= backspaces <= 63 + correction = correction[i:] + bs_count = [backspaces + 128] + data = bs_count + list(bytes(correction, 'ascii')) + [0] + + entry = {'data': data, 'links': [], 'byte_offset': 0} + table.append(entry) + elif len(trie_node) == 1: # Handle trie node with a single child. + c, trie_node = next(iter(trie_node.items())) + entry = {'chars': c, 'byte_offset': 0} + + # It's common for a trie to have long chains of single-child nodes. We + # find the whole chain so that we can serialize it more efficiently. + while len(trie_node) == 1 and 'LEAF' not in trie_node: + c, trie_node = next(iter(trie_node.items())) + entry['chars'] += c + + table.append(entry) + entry['links'] = [traverse(trie_node)] + else: # Handle trie node with multiple children. + entry = {'chars': ''.join(sorted(trie_node.keys())), 'byte_offset': 0} + table.append(entry) + entry['links'] = [traverse(trie_node[c]) for c in entry['chars']] + return entry + + traverse(trie) + + def serialize(e: Dict[str, Any]) -> List[int]: + if not e['links']: # Handle a leaf table entry. + return e['data'] + elif len(e['links']) == 1: # Handle a chain table entry. + return [TYPO_CHARS[c] for c in e['chars']] + [0] # + encode_link(e['links'][0])) + else: # Handle a branch table entry. + data = [] + for c, link in zip(e['chars'], e['links']): + data += [TYPO_CHARS[c] | (0 if data else 64)] + encode_link(link) + return data + [0] + + byte_offset = 0 + for e in table: # To encode links, first compute byte offset of each entry. + e['byte_offset'] = byte_offset + byte_offset += len(serialize(e)) + assert 0 <= byte_offset <= 0xffff + + return [b for e in table for b in serialize(e)] # Serialize final table. + + +def encode_link(link: Dict[str, Any]) -> List[int]: + """Encodes a node link as two bytes.""" + byte_offset = link['byte_offset'] + if not (0 <= byte_offset <= 0xffff): + cli.log.error('{fg_red}Error:{fg_reset} The autocorrection table is too large, a node link exceeds 64KB limit. Try reducing the autocorrection dict to fewer entries.') + sys.exit(1) + return [byte_offset & 255, byte_offset >> 8] + + +def write_generated_code(autocorrections: List[Tuple[str, str]], data: List[int], file_name: str) -> None: + """Writes autocorrection data as generated C code to `file_name`. + Args: + autocorrections: List of (typo, correction) tuples. + data: List of ints in 0-255, the serialized trie. + file_name: String, path of the output C file. + """ + assert all(0 <= b <= 255 for b in data) + + def typo_len(e: Tuple[str, str]) -> int: + return len(e[0]) + + min_typo = min(autocorrections, key=typo_len)[0] + max_typo = max(autocorrections, key=typo_len)[0] + generated_code = ''.join([ + '// Generated code.\n\n', f'// Autocorrection dictionary ({len(autocorrections)} entries):\n', ''.join(sorted(f'// {typo:<{len(max_typo)}} -> {correction}\n' for typo, correction in autocorrections)), + f'\n#define AUTOCORRECT_MIN_LENGTH {len(min_typo)} // "{min_typo}"\n', f'#define AUTOCORRECT_MAX_LENGTH {len(max_typo)} // "{max_typo}"\n\n', f'#define DICTIONARY_SIZE {len(data)}\n\n', + textwrap.fill('static const uint8_t autocorrect_data[DICTIONARY_SIZE] PROGMEM = {%s};' % (', '.join(map(str, data))), width=120, subsequent_indent=' '), '\n\n' + ]) + + with open(file_name, 'wt') as f: + f.write(generated_code) + + +@cli.argument('filename', default='autocorrect_dict.txt', help='The autocorrection database file') +@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') +@cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.') +@cli.argument('-o', '--output', arg_only=True, type=qmk.path.normpath, help='File to write to') +@cli.subcommand('Generate the autocorrection data file from a dictionary file.') +def generate_autocorrect_data(cli): + autocorrections = parse_file(cli.args.filename) + trie = make_trie(autocorrections) + data = serialize_trie(autocorrections, trie) + # Environment processing + if cli.args.output == '-': + cli.args.output = None + + if cli.args.output: + cli.args.output.parent.mkdir(parents=True, exist_ok=True) + cli.log.info('Creating autocorrect database at {fg_cyan}%s', cli.args.output) + write_generated_code(autocorrections, data, cli.args.output) + + else: + current_keyboard = cli.args.keyboard or cli.config.user.keyboard or cli.config.generate_autocorrect_data.keyboard + current_keymap = cli.args.keymap or cli.config.user.keymap or cli.config.generate_autocorrect_data.keymap + + if current_keyboard and current_keymap: + filename = locate_keymap(current_keyboard, current_keymap).parent / 'autocorrect_data.h' + cli.log.info('Creating autocorrect database at {fg_cyan}%s', filename) + write_generated_code(autocorrections, data, filename) + + else: + write_generated_code(autocorrections, data, 'autocorrect_data.h') + + cli.log.info('Processed %d autocorrection entries to table with %d bytes.', len(autocorrections), len(data)) diff --git a/lib/python/qmk/cli/generate/config_h.py b/lib/python/qmk/cli/generate/config_h.py index a26dcdf7d7..31b8d70635 100755 --- a/lib/python/qmk/cli/generate/config_h.py +++ b/lib/python/qmk/cli/generate/config_h.py @@ -1,18 +1,27 @@ """Used by the make system to generate info_config.h from info.json. """ from pathlib import Path - from dotty_dict import dotty + +from argcomplete.completers import FilesCompleter from milc import cli -from qmk.info import info_json, keymap_json_config +from qmk.info import info_json from qmk.json_schema import json_load from qmk.keyboard import keyboard_completer, keyboard_folder -from qmk.commands import dump_lines -from qmk.path import normpath +from qmk.commands import dump_lines, parse_configurator_json +from qmk.path import normpath, FileType from qmk.constants import GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE +def generate_define(define, value=None): + value = f' {value}' if value is not None else '' + return f""" +#ifndef {define} +# define {define}{value} +#endif // {define}""" + + def direct_pins(direct_pins, postfix): """Return the config.h lines that set the direct pins. """ @@ -22,11 +31,7 @@ def direct_pins(direct_pins, postfix): cols = ','.join(map(str, [col or 'NO_PIN' for col in row])) rows.append('{' + cols + '}') - return f""" -#ifndef DIRECT_PINS{postfix} -# define DIRECT_PINS{postfix} {{ {", ".join(rows)} }} -#endif // DIRECT_PINS{postfix} -""" + return generate_define(f'DIRECT_PINS{postfix}', f'{{ {", ".join(rows)} }}') def pin_array(define, pins, postfix): @@ -34,11 +39,7 @@ def pin_array(define, pins, postfix): """ pin_array = ', '.join(map(str, [pin or 'NO_PIN' for pin in pins])) - return f""" -#ifndef {define}_PINS{postfix} -# define {define}_PINS{postfix} {{ {pin_array} }} -#endif // {define}_PINS{postfix} -""" + return generate_define(f'{define}_PINS{postfix}', f'{{ {pin_array} }}') def matrix_pins(matrix_pins, postfix=''): @@ -62,24 +63,14 @@ def generate_matrix_size(kb_info_json, config_h_lines): """Add the matrix size to the config.h. """ if 'matrix_pins' in kb_info_json: - col_count = kb_info_json['matrix_size']['cols'] - row_count = kb_info_json['matrix_size']['rows'] - - config_h_lines.append(f""" -#ifndef MATRIX_COLS -# define MATRIX_COLS {col_count} -#endif // MATRIX_COLS - -#ifndef MATRIX_ROWS -# define MATRIX_ROWS {row_count} -#endif // MATRIX_ROWS -""") + config_h_lines.append(generate_define('MATRIX_COLS', kb_info_json['matrix_size']['cols'])) + config_h_lines.append(generate_define('MATRIX_ROWS', kb_info_json['matrix_size']['rows'])) def generate_config_items(kb_info_json, config_h_lines): """Iterate through the info_config map to generate basic config values. """ - info_config_map = json_load(Path('data/mappings/info_config.json')) + info_config_map = json_load(Path('data/mappings/info_config.hjson')) for config_key, info_dict in info_config_map.items(): info_key = info_dict['info_key'] @@ -95,43 +86,23 @@ def generate_config_items(kb_info_json, config_h_lines): continue if key_type.startswith('array.array'): - config_h_lines.append('') - config_h_lines.append(f'#ifndef {config_key}') - config_h_lines.append(f'# define {config_key} {{ {", ".join(["{" + ",".join(list(map(str, x))) + "}" for x in config_value])} }}') - config_h_lines.append(f'#endif // {config_key}') + config_h_lines.append(generate_define(config_key, f'{{ {", ".join(["{" + ",".join(list(map(str, x))) + "}" for x in config_value])} }}')) elif key_type.startswith('array'): - config_h_lines.append('') - config_h_lines.append(f'#ifndef {config_key}') - config_h_lines.append(f'# define {config_key} {{ {", ".join(map(str, config_value))} }}') - config_h_lines.append(f'#endif // {config_key}') + config_h_lines.append(generate_define(config_key, f'{{ {", ".join(map(str, config_value))} }}')) elif key_type == 'bool': if config_value: - config_h_lines.append('') - config_h_lines.append(f'#ifndef {config_key}') - config_h_lines.append(f'# define {config_key}') - config_h_lines.append(f'#endif // {config_key}') + config_h_lines.append(generate_define(config_key)) elif key_type == 'mapping': for key, value in config_value.items(): - config_h_lines.append('') - config_h_lines.append(f'#ifndef {key}') - config_h_lines.append(f'# define {key} {value}') - config_h_lines.append(f'#endif // {key}') + config_h_lines.append(generate_define(key, value)) elif key_type == 'str': - config_h_lines.append('') - config_h_lines.append(f'#ifndef {config_key}') - config_h_lines.append(f'# define {config_key} "{config_value}"') - config_h_lines.append(f'#endif // {config_key}') + escaped_str = config_value.replace('\\', '\\\\').replace('"', '\\"') + config_h_lines.append(generate_define(config_key, f'"{escaped_str}"')) elif key_type == 'bcd_version': (major, minor, revision) = config_value.split('.') - config_h_lines.append('') - config_h_lines.append(f'#ifndef {config_key}') - config_h_lines.append(f'# define {config_key} 0x{major.zfill(2)}{minor}{revision}') - config_h_lines.append(f'#endif // {config_key}') + config_h_lines.append(generate_define(config_key, f'0x{major.zfill(2)}{minor}{revision}')) else: - config_h_lines.append('') - config_h_lines.append(f'#ifndef {config_key}') - config_h_lines.append(f'# define {config_key} {config_value}') - config_h_lines.append(f'#endif // {config_key}') + config_h_lines.append(generate_define(config_key, config_value)) def generate_encoder_config(encoder_json, config_h_lines, postfix=''): @@ -144,24 +115,15 @@ def generate_encoder_config(encoder_json, config_h_lines, postfix=''): b_pads.append(encoder["pin_b"]) resolutions.append(encoder.get("resolution", None)) - config_h_lines.append(f'#ifndef ENCODERS_PAD_A{postfix}') - config_h_lines.append(f'# define ENCODERS_PAD_A{postfix} {{ { ", ".join(a_pads) } }}') - config_h_lines.append(f'#endif // ENCODERS_PAD_A{postfix}') - - config_h_lines.append(f'#ifndef ENCODERS_PAD_B{postfix}') - config_h_lines.append(f'# define ENCODERS_PAD_B{postfix} {{ { ", ".join(b_pads) } }}') - config_h_lines.append(f'#endif // ENCODERS_PAD_B{postfix}') + config_h_lines.append(generate_define(f'ENCODERS_PAD_A{postfix}', f'{{ {", ".join(a_pads)} }}')) + config_h_lines.append(generate_define(f'ENCODERS_PAD_B{postfix}', f'{{ {", ".join(b_pads)} }}')) if None in resolutions: cli.log.debug("Unable to generate ENCODER_RESOLUTION configuration") elif len(set(resolutions)) == 1: - config_h_lines.append(f'#ifndef ENCODER_RESOLUTION{postfix}') - config_h_lines.append(f'# define ENCODER_RESOLUTION{postfix} { resolutions[0] }') - config_h_lines.append(f'#endif // ENCODER_RESOLUTION{postfix}') + config_h_lines.append(generate_define(f'ENCODER_RESOLUTION{postfix}', resolutions[0])) else: - config_h_lines.append(f'#ifndef ENCODER_RESOLUTIONS{postfix}') - config_h_lines.append(f'# define ENCODER_RESOLUTIONS{postfix} {{ { ", ".join(map(str,resolutions)) } }}') - config_h_lines.append(f'#endif // ENCODER_RESOLUTIONS{postfix}') + config_h_lines.append(generate_define(f'ENCODER_RESOLUTIONS{postfix}', f'{{ {", ".join(map(str,resolutions))} }}')) def generate_split_config(kb_info_json, config_h_lines): @@ -170,35 +132,23 @@ def generate_split_config(kb_info_json, config_h_lines): if kb_info_json['split']['primary'] in ('left', 'right'): config_h_lines.append('') config_h_lines.append('#ifndef MASTER_LEFT') - config_h_lines.append('# ifndef MASTER_RIGHT') + config_h_lines.append('# ifndef MASTER_RIGHT') if kb_info_json['split']['primary'] == 'left': - config_h_lines.append('# define MASTER_LEFT') + config_h_lines.append('# define MASTER_LEFT') elif kb_info_json['split']['primary'] == 'right': - config_h_lines.append('# define MASTER_RIGHT') - config_h_lines.append('# endif // MASTER_RIGHT') + config_h_lines.append('# define MASTER_RIGHT') + config_h_lines.append('# endif // MASTER_RIGHT') config_h_lines.append('#endif // MASTER_LEFT') elif kb_info_json['split']['primary'] == 'pin': - config_h_lines.append('') - config_h_lines.append('#ifndef SPLIT_HAND_PIN') - config_h_lines.append('# define SPLIT_HAND_PIN') - config_h_lines.append('#endif // SPLIT_HAND_PIN') + config_h_lines.append(generate_define('SPLIT_HAND_PIN')) elif kb_info_json['split']['primary'] == 'matrix_grid': - config_h_lines.append('') - config_h_lines.append('#ifndef SPLIT_HAND_MATRIX_GRID') - config_h_lines.append('# define SPLIT_HAND_MATRIX_GRID {%s}' % (','.join(kb_info_json["split"]["matrix_grid"],))) - config_h_lines.append('#endif // SPLIT_HAND_MATRIX_GRID') + config_h_lines.append(generate_define('SPLIT_HAND_MATRIX_GRID', f'{{ {",".join(kb_info_json["split"]["matrix_grid"])} }}')) elif kb_info_json['split']['primary'] == 'eeprom': - config_h_lines.append('') - config_h_lines.append('#ifndef EE_HANDS') - config_h_lines.append('# define EE_HANDS') - config_h_lines.append('#endif // EE_HANDS') + config_h_lines.append(generate_define('EE_HANDS')) if 'protocol' in kb_info_json['split'].get('transport', {}): if kb_info_json['split']['transport']['protocol'] == 'i2c': - config_h_lines.append('') - config_h_lines.append('#ifndef USE_I2C') - config_h_lines.append('# define USE_I2C') - config_h_lines.append('#endif // USE_I2C') + config_h_lines.append(generate_define('USE_I2C')) if 'right' in kb_info_json['split'].get('matrix_pins', {}): config_h_lines.append(matrix_pins(kb_info_json['split']['matrix_pins']['right'], '_RIGHT')) @@ -207,19 +157,30 @@ def generate_split_config(kb_info_json, config_h_lines): generate_encoder_config(kb_info_json['split']['encoder']['right'], config_h_lines, '_RIGHT') +def generate_led_animations_config(led_feature_json, config_h_lines, prefix): + for animation in led_feature_json.get('animations', {}): + if led_feature_json['animations'][animation]: + config_h_lines.append(generate_define(f'{prefix}{animation.upper()}')) + + +@cli.argument('filename', nargs='?', arg_only=True, type=FileType('r'), completer=FilesCompleter('.json'), help='A configurator export JSON to be compiled and flashed or a pre-compiled binary firmware file (bin/hex) to be flashed.') @cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to') @cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages") -@cli.argument('-kb', '--keyboard', arg_only=True, type=keyboard_folder, completer=keyboard_completer, required=True, help='Keyboard to generate config.h for.') -@cli.argument('-km', '--keymap', arg_only=True, help='Keymap to generate config.h for.') +@cli.argument('-kb', '--keyboard', arg_only=True, type=keyboard_folder, completer=keyboard_completer, help='Keyboard to generate config.h for.') @cli.subcommand('Used by the make system to generate info_config.h from info.json', hidden=True) def generate_config_h(cli): """Generates the info_config.h file. """ # Determine our keyboard/keymap - if cli.args.keymap: - kb_info_json = dotty(keymap_json_config(cli.args.keyboard, cli.args.keymap)) - else: + if cli.args.filename: + user_keymap = parse_configurator_json(cli.args.filename) + kb_info_json = dotty(user_keymap.get('config', {})) + elif cli.args.keyboard: kb_info_json = dotty(info_json(cli.args.keyboard)) + else: + cli.log.error('You must supply a configurator export or `--keyboard`.') + cli.subcommands['generate-config-h'].print_help() + return False # Build the info_config.h file. config_h_lines = [GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE, '#pragma once'] @@ -237,5 +198,14 @@ def generate_config_h(cli): if 'split' in kb_info_json: generate_split_config(kb_info_json, config_h_lines) + if 'led_matrix' in kb_info_json: + generate_led_animations_config(kb_info_json['led_matrix'], config_h_lines, 'ENABLE_LED_MATRIX_') + + if 'rgb_matrix' in kb_info_json: + generate_led_animations_config(kb_info_json['rgb_matrix'], config_h_lines, 'ENABLE_RGB_MATRIX_') + + if 'rgblight' in kb_info_json: + generate_led_animations_config(kb_info_json['rgblight'], config_h_lines, 'RGBLIGHT_EFFECT_') + # Show the results dump_lines(cli.args.output, config_h_lines, cli.args.quiet) diff --git a/lib/python/qmk/cli/generate/keyboard_h.py b/lib/python/qmk/cli/generate/keyboard_h.py index 54ddb4cffd..910bd6a08d 100755 --- a/lib/python/qmk/cli/generate/keyboard_h.py +++ b/lib/python/qmk/cli/generate/keyboard_h.py @@ -1,33 +1,72 @@ """Used by the make system to generate keyboard.h from info.json. """ +from pathlib import Path + from milc import cli +from qmk.path import normpath from qmk.info import info_json from qmk.commands import dump_lines from qmk.keyboard import keyboard_completer, keyboard_folder -from qmk.path import normpath -from qmk.constants import GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE +from qmk.constants import COL_LETTERS, ROW_LETTERS, GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE -def would_populate_layout_h(keyboard): - """Detect if a given keyboard is doing data driven layouts +def _generate_layouts(keyboard): + """Generates the layouts.h file. """ # Build the info.json file kb_info_json = info_json(keyboard) + if 'matrix_size' not in kb_info_json: + cli.log.error(f'{keyboard}: Invalid matrix config.') + return [] + + col_num = kb_info_json['matrix_size']['cols'] + row_num = kb_info_json['matrix_size']['rows'] + + lines = [] for layout_name in kb_info_json['layouts']: if kb_info_json['layouts'][layout_name]['c_macro']: continue if 'matrix' not in kb_info_json['layouts'][layout_name]['layout'][0]: - cli.log.debug('%s/%s: No matrix data!', keyboard, layout_name) + cli.log.debug(f'{keyboard}/{layout_name}: No matrix data!') continue - return True + layout_keys = [] + layout_matrix = [['KC_NO' for i in range(col_num)] for i in range(row_num)] + + for i, key in enumerate(kb_info_json['layouts'][layout_name]['layout']): + row = key['matrix'][0] + col = key['matrix'][1] + identifier = 'k%s%s' % (ROW_LETTERS[row], COL_LETTERS[col]) - return False + try: + layout_matrix[row][col] = identifier + layout_keys.append(identifier) + except IndexError: + key_name = key.get('label', identifier) + cli.log.error(f'Matrix data out of bounds for layout {layout_name} at index {i} ({key_name}): [{row}, {col}]') + return [] + lines.append('') + lines.append('#define %s(%s) {\\' % (layout_name, ', '.join(layout_keys))) + rows = ', \\\n'.join(['\t {' + ', '.join(row) + '}' for row in layout_matrix]) + rows += ' \\' + lines.append(rows) + lines.append('}') + + for alias, target in kb_info_json.get('layout_aliases', {}).items(): + lines.append('') + lines.append(f'#ifndef {alias}') + lines.append(f'# define {alias} {target}') + lines.append('#endif') + + return lines + + +@cli.argument('-i', '--include', nargs='?', arg_only=True, help='Optional file to include') @cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to') @cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages") @cli.argument('-kb', '--keyboard', arg_only=True, type=keyboard_folder, completer=keyboard_completer, required=True, help='Keyboard to generate keyboard.h for.') @@ -35,13 +74,23 @@ def would_populate_layout_h(keyboard): def generate_keyboard_h(cli): """Generates the keyboard.h file. """ - has_layout_h = would_populate_layout_h(cli.args.keyboard) + keyboard_h = cli.args.include + dd_layouts = _generate_layouts(cli.args.keyboard) + valid_config = dd_layouts or keyboard_h # Build the layouts.h file. keyboard_h_lines = [GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE, '#pragma once', '#include "quantum.h"'] - if not has_layout_h: - keyboard_h_lines.append('#error("<keyboard>.h is only optional for data driven keyboards - kb.h == bad times")') + keyboard_h_lines.append('') + keyboard_h_lines.append('// Layout content') + if dd_layouts: + keyboard_h_lines.extend(dd_layouts) + if keyboard_h: + keyboard_h_lines.append(f'#include "{Path(keyboard_h).name}"') + + # Protect against poorly configured keyboards + if not valid_config: + keyboard_h_lines.append('#error("<keyboard>.h is required unless your keyboard uses data-driven configuration. Please rename your keyboard\'s header file to <keyboard>.h")') # Show the results dump_lines(cli.args.output, keyboard_h_lines, cli.args.quiet) diff --git a/lib/python/qmk/cli/generate/keycodes.py b/lib/python/qmk/cli/generate/keycodes.py new file mode 100644 index 0000000000..29b7db3c80 --- /dev/null +++ b/lib/python/qmk/cli/generate/keycodes.py @@ -0,0 +1,88 @@ +"""Used by the make system to generate keycodes.h from keycodes_{version}.json +""" +from milc import cli + +from qmk.constants import GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE +from qmk.commands import dump_lines +from qmk.path import normpath +from qmk.keycodes import load_spec + + +def _generate_ranges(lines, keycodes): + lines.append('') + lines.append('enum qk_keycode_ranges {') + lines.append('// Ranges') + for key, value in keycodes["ranges"].items(): + lo, mask = map(lambda x: int(x, 16), key.split("/")) + hi = lo + mask + define = value.get("define") + lines.append(f' {define.ljust(30)} = 0x{lo:04X},') + lines.append(f' {(define + "_MAX").ljust(30)} = 0x{hi:04X},') + lines.append('};') + + +def _generate_defines(lines, keycodes): + lines.append('') + lines.append('enum qk_keycode_defines {') + lines.append('// Keycodes') + for key, value in keycodes["keycodes"].items(): + lines.append(f' {value.get("key")} = {key},') + + lines.append('') + lines.append('// Alias') + for key, value in keycodes["keycodes"].items(): + temp = value.get("key") + for alias in value.get("aliases", []): + lines.append(f' {alias.ljust(10)} = {temp},') + + lines.append('};') + + +def _generate_helpers(lines, keycodes): + lines.append('') + lines.append('// Range Helpers') + for value in keycodes["ranges"].values(): + define = value.get("define") + lines.append(f'#define IS_{define}(code) ((code) >= {define} && (code) <= {define + "_MAX"})') + + # extract min/max + temp = {} + for key, value in keycodes["keycodes"].items(): + group = value.get('group', None) + if not group: + continue + if group not in temp: + temp[group] = [0xFFFF, 0] + key = int(key, 16) + if key < temp[group][0]: + temp[group][0] = key + if key > temp[group][1]: + temp[group][1] = key + + lines.append('') + lines.append('// Group Helpers') + for group, codes in temp.items(): + lo = keycodes["keycodes"][f'0x{codes[0]:04X}']['key'] + hi = keycodes["keycodes"][f'0x{codes[1]:04X}']['key'] + lines.append(f'#define IS_{ group.upper() }_KEYCODE(code) ((code) >= {lo} && (code) <= {hi})') + + +@cli.argument('-v', '--version', arg_only=True, required=True, help='Version of keycodes to generate.') +@cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to') +@cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages") +@cli.subcommand('Used by the make system to generate keycodes.h from keycodes_{version}.json', hidden=True) +def generate_keycodes(cli): + """Generates the keycodes.h file. + """ + + # Build the keycodes.h file. + keycodes_h_lines = [GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE, '#pragma once', '// clang-format off'] + + keycodes = load_spec(cli.args.version) + + _generate_ranges(keycodes_h_lines, keycodes) + _generate_defines(keycodes_h_lines, keycodes) + _generate_helpers(keycodes_h_lines, keycodes) + + # Show the results + dump_lines(cli.args.output, keycodes_h_lines, cli.args.quiet) diff --git a/lib/python/qmk/cli/generate/layouts.py b/lib/python/qmk/cli/generate/layouts.py deleted file mode 100755 index 193633baf6..0000000000 --- a/lib/python/qmk/cli/generate/layouts.py +++ /dev/null @@ -1,90 +0,0 @@ -"""Used by the make system to generate layouts.h from info.json. -""" -from milc import cli - -from qmk.constants import COL_LETTERS, ROW_LETTERS, GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE -from qmk.decorators import automagic_keyboard, automagic_keymap -from qmk.info import info_json -from qmk.keyboard import keyboard_completer, keyboard_folder -from qmk.path import is_keyboard, normpath -from qmk.commands import dump_lines - -usb_properties = { - 'vid': 'VENDOR_ID', - 'pid': 'PRODUCT_ID', - 'device_ver': 'DEVICE_VER', -} - - -@cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to') -@cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages") -@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='Keyboard to generate config.h for.') -@cli.subcommand('Used by the make system to generate layouts.h from info.json', hidden=True) -@automagic_keyboard -@automagic_keymap -def generate_layouts(cli): - """Generates the layouts.h file. - """ - # Determine our keyboard(s) - if not cli.config.generate_layouts.keyboard: - cli.log.error('Missing parameter: --keyboard') - cli.subcommands['info'].print_help() - return False - - if not is_keyboard(cli.config.generate_layouts.keyboard): - cli.log.error('Invalid keyboard: "%s"', cli.config.generate_layouts.keyboard) - return False - - # Build the info.json file - kb_info_json = info_json(cli.config.generate_layouts.keyboard) - - # Build the layouts.h file. - layouts_h_lines = [GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE, '#pragma once'] - - if 'matrix_size' not in kb_info_json: - cli.log.error('%s: Invalid matrix config.', cli.config.generate_layouts.keyboard) - return False - - col_num = kb_info_json['matrix_size']['cols'] - row_num = kb_info_json['matrix_size']['rows'] - - for layout_name in kb_info_json['layouts']: - if kb_info_json['layouts'][layout_name]['c_macro']: - continue - - if 'matrix' not in kb_info_json['layouts'][layout_name]['layout'][0]: - cli.log.debug('%s/%s: No matrix data!', cli.config.generate_layouts.keyboard, layout_name) - continue - - layout_keys = [] - layout_matrix = [['KC_NO' for i in range(col_num)] for i in range(row_num)] - - for i, key in enumerate(kb_info_json['layouts'][layout_name]['layout']): - row = key['matrix'][0] - col = key['matrix'][1] - identifier = 'k%s%s' % (ROW_LETTERS[row], COL_LETTERS[col]) - - try: - layout_matrix[row][col] = identifier - layout_keys.append(identifier) - except IndexError: - key_name = key.get('label', identifier) - cli.log.error('Matrix data out of bounds for layout %s at index %s (%s): %s, %s', layout_name, i, key_name, row, col) - return False - - layouts_h_lines.append('') - layouts_h_lines.append('#define %s(%s) {\\' % (layout_name, ', '.join(layout_keys))) - - rows = ', \\\n'.join(['\t {' + ', '.join(row) + '}' for row in layout_matrix]) - rows += ' \\' - layouts_h_lines.append(rows) - layouts_h_lines.append('}') - - for alias, target in kb_info_json.get('layout_aliases', {}).items(): - layouts_h_lines.append('') - layouts_h_lines.append(f'#ifndef {alias}') - layouts_h_lines.append(f'# define {alias} {target}') - layouts_h_lines.append('#endif') - - # Show the results - dump_lines(cli.args.output, layouts_h_lines, cli.args.quiet) diff --git a/lib/python/qmk/cli/generate/rules_mk.py b/lib/python/qmk/cli/generate/rules_mk.py index 9623d00fb5..fc272da6c6 100755 --- a/lib/python/qmk/cli/generate/rules_mk.py +++ b/lib/python/qmk/cli/generate/rules_mk.py @@ -1,15 +1,16 @@ """Used by the make system to generate a rules.mk """ from pathlib import Path - from dotty_dict import dotty + +from argcomplete.completers import FilesCompleter from milc import cli -from qmk.info import info_json, keymap_json_config +from qmk.info import info_json from qmk.json_schema import json_load from qmk.keyboard import keyboard_completer, keyboard_folder -from qmk.commands import dump_lines -from qmk.path import normpath +from qmk.commands import dump_lines, parse_configurator_json +from qmk.path import normpath, FileType from qmk.constants import GPL2_HEADER_SH_LIKE, GENERATED_HEADER_SH_LIKE @@ -39,22 +40,29 @@ def process_mapping_rule(kb_info_json, rules_key, info_dict): return f'{rules_key} ?= {rules_value}' +@cli.argument('filename', nargs='?', arg_only=True, type=FileType('r'), completer=FilesCompleter('.json'), help='A configurator export JSON to be compiled and flashed or a pre-compiled binary firmware file (bin/hex) to be flashed.') @cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to') @cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages") @cli.argument('-e', '--escape', arg_only=True, action='store_true', help="Escape spaces in quiet mode") -@cli.argument('-kb', '--keyboard', arg_only=True, type=keyboard_folder, completer=keyboard_completer, required=True, help='Keyboard to generate rules.mk for.') -@cli.argument('-km', '--keymap', arg_only=True, help='Keymap to generate rules.mk for.') +@cli.argument('-kb', '--keyboard', arg_only=True, type=keyboard_folder, completer=keyboard_completer, help='Keyboard to generate rules.mk for.') @cli.subcommand('Used by the make system to generate rules.mk from info.json', hidden=True) def generate_rules_mk(cli): """Generates a rules.mk file from info.json. """ + converter = None # Determine our keyboard/keymap - if cli.args.keymap: - kb_info_json = dotty(keymap_json_config(cli.args.keyboard, cli.args.keymap)) - else: + if cli.args.filename: + user_keymap = parse_configurator_json(cli.args.filename) + kb_info_json = dotty(user_keymap.get('config', {})) + converter = user_keymap.get('converter', None) + elif cli.args.keyboard: kb_info_json = dotty(info_json(cli.args.keyboard)) + else: + cli.log.error('You must supply a configurator export or `--keyboard`.') + cli.subcommands['generate-rules-mk'].print_help() + return False - info_rules_map = json_load(Path('data/mappings/info_rules.json')) + info_rules_map = json_load(Path('data/mappings/info_rules.hjson')) rules_mk_lines = [GPL2_HEADER_SH_LIKE, GENERATED_HEADER_SH_LIKE] # Iterate through the info_rules map to generate basic rules @@ -82,6 +90,9 @@ def generate_rules_mk(cli): else: rules_mk_lines.append('CUSTOM_MATRIX ?= yes') + if converter: + rules_mk_lines.append(f'CONVERT_TO ?= {converter}') + # Show the results dump_lines(cli.args.output, rules_mk_lines) diff --git a/lib/python/qmk/cli/new/keyboard.py b/lib/python/qmk/cli/new/keyboard.py index 8d4def1bef..251ad919dd 100644 --- a/lib/python/qmk/cli/new/keyboard.py +++ b/lib/python/qmk/cli/new/keyboard.py @@ -210,7 +210,7 @@ def new_keyboard(cli): # Preprocess any development_board presets if mcu in dev_boards: - defaults_map = json_load(Path('data/mappings/defaults.json')) + defaults_map = json_load(Path('data/mappings/defaults.hjson')) board = defaults_map['development_board'][mcu] mcu = board['processor'] diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py index 9c0a5dce56..5561a354c5 100644 --- a/lib/python/qmk/commands.py +++ b/lib/python/qmk/commands.py @@ -2,13 +2,13 @@ """ import os import sys +import json import shutil from pathlib import Path from milc import cli import jsonschema -import qmk.keymap from qmk.constants import KEYBOARD_OUTPUT_PREFIX from qmk.json_schema import json_load, validate @@ -107,7 +107,7 @@ def get_make_parallel_args(parallel=1): return parallel_args -def compile_configurator_json(user_keymap, bootloader=None, parallel=1, **env_vars): +def compile_configurator_json(user_keymap, bootloader=None, parallel=1, clean=False, **env_vars): """Convert a configurator export JSON file into a C file and then compile it. Args: @@ -129,17 +129,32 @@ def compile_configurator_json(user_keymap, bootloader=None, parallel=1, **env_va # e.g.: qmk compile - < keyboards/clueboard/california/keymaps/default/keymap.json user_keymap["keymap"] = user_keymap.get("keymap", "default_json") - # Write the keymap.c file keyboard_filesafe = user_keymap['keyboard'].replace('/', '_') target = f'{keyboard_filesafe}_{user_keymap["keymap"]}' keyboard_output = Path(f'{KEYBOARD_OUTPUT_PREFIX}{keyboard_filesafe}') keymap_output = Path(f'{keyboard_output}_{user_keymap["keymap"]}') - c_text = qmk.keymap.generate_c(user_keymap) keymap_dir = keymap_output / 'src' - keymap_c = keymap_dir / 'keymap.c' + keymap_json = keymap_dir / 'keymap.json' + if clean: + if keyboard_output.exists(): + shutil.rmtree(keyboard_output) + if keymap_output.exists(): + shutil.rmtree(keymap_output) + + # begin with making the deepest folder in the tree keymap_dir.mkdir(exist_ok=True, parents=True) - keymap_c.write_text(c_text) + + # Compare minified to ensure consistent comparison + new_content = json.dumps(user_keymap, separators=(',', ':')) + if keymap_json.exists(): + old_content = json.dumps(json.loads(keymap_json.read_text(encoding='utf-8')), separators=(',', ':')) + if old_content == new_content: + new_content = None + + # Write the keymap.json file if different + if new_content: + keymap_json.write_text(new_content, encoding='utf-8') # Return a command that can be run to make the keymap and flash if given verbose = 'true' if cli.config.general.verbose else 'false' @@ -175,7 +190,7 @@ def compile_configurator_json(user_keymap, bootloader=None, parallel=1, **env_va f'MAIN_KEYMAP_PATH_3={keymap_output}', f'MAIN_KEYMAP_PATH_4={keymap_output}', f'MAIN_KEYMAP_PATH_5={keymap_output}', - f'KEYMAP_C={keymap_c}', + f'KEYMAP_JSON={keymap_json}', f'KEYMAP_PATH={keymap_dir}', f'VERBOSE={verbose}', f'COLOR={color}', @@ -199,7 +214,7 @@ def parse_configurator_json(configurator_file): exit(1) orig_keyboard = user_keymap['keyboard'] - aliases = json_load(Path('data/mappings/keyboard_aliases.json')) + aliases = json_load(Path('data/mappings/keyboard_aliases.hjson')) if orig_keyboard in aliases: if 'target' in aliases[orig_keyboard]: @@ -211,6 +226,19 @@ def parse_configurator_json(configurator_file): return user_keymap +def build_environment(args): + """Common processing for cli.args.env + """ + envs = {} + for env in args: + if '=' in env: + key, value = env.split('=', 1) + envs[key] = value + else: + cli.log.warning('Invalid environment variable: %s', env) + return envs + + def in_virtualenv(): """Check if running inside a virtualenv. Based on https://stackoverflow.com/a/1883251 diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py index 7460d84ad3..5dc8b9c5fe 100644 --- a/lib/python/qmk/info.py +++ b/lib/python/qmk/info.py @@ -437,19 +437,6 @@ def _extract_matrix_info(info_data, config_c): return info_data -# TODO: kill off usb.device_ver in favor of usb.device_version -def _extract_device_version(info_data): - if info_data.get('usb'): - if info_data['usb'].get('device_version') and not info_data['usb'].get('device_ver'): - (major, minor, revision) = info_data['usb']['device_version'].split('.', 3) - info_data['usb']['device_ver'] = f'0x{major.zfill(2)}{minor}{revision}' - if not info_data['usb'].get('device_version') and info_data['usb'].get('device_ver'): - major = int(info_data['usb']['device_ver'][2:4]) - minor = int(info_data['usb']['device_ver'][4]) - revision = int(info_data['usb']['device_ver'][5]) - info_data['usb']['device_version'] = f'{major}.{minor}.{revision}' - - def _config_to_json(key_type, config_value): """Convert config value using spec """ @@ -479,7 +466,7 @@ def _config_to_json(key_type, config_value): return int(config_value) elif key_type == 'str': - return config_value.strip('"') + return config_value.strip('"').replace('\\"', '"').replace('\\\\', '\\') elif key_type == 'bcd_version': major = int(config_value[2:4]) @@ -496,7 +483,7 @@ def _extract_config_h(info_data, config_c): """ # Pull in data from the json map dotty_info = dotty(info_data) - info_config_map = json_load(Path('data/mappings/info_config.json')) + info_config_map = json_load(Path('data/mappings/info_config.hjson')) for config_key, info_dict in info_config_map.items(): info_key = info_dict['info_key'] @@ -535,7 +522,6 @@ def _extract_config_h(info_data, config_c): _extract_split_right_pins(info_data, config_c) _extract_encoders(info_data, config_c) _extract_split_encoders(info_data, config_c) - _extract_device_version(info_data) return info_data @@ -543,7 +529,7 @@ def _extract_config_h(info_data, config_c): def _process_defaults(info_data): """Process any additional defaults based on currently discovered information """ - defaults_map = json_load(Path('data/mappings/defaults.json')) + defaults_map = json_load(Path('data/mappings/defaults.hjson')) for default_type in defaults_map.keys(): thing_map = defaults_map[default_type] if default_type in info_data: @@ -569,7 +555,7 @@ def _extract_rules_mk(info_data, rules): # Pull in data from the json map dotty_info = dotty(info_data) - info_rules_map = json_load(Path('data/mappings/info_rules.json')) + info_rules_map = json_load(Path('data/mappings/info_rules.hjson')) for rules_key, info_dict in info_rules_map.items(): info_key = info_dict['info_key'] @@ -627,20 +613,24 @@ def _extract_led_config(info_data, keyboard): cols = info_data['matrix_size']['cols'] rows = info_data['matrix_size']['rows'] - # Assume what feature owns g_led_config - feature = "rgb_matrix" - if info_data.get("features", {}).get("led_matrix", False): + # Determine what feature owns g_led_config + features = info_data.get("features", {}) + feature = None + if features.get("rgb_matrix", False): + feature = "rgb_matrix" + elif features.get("led_matrix", False): feature = "led_matrix" - # Process - for file in find_keyboard_c(keyboard): - try: - ret = find_led_config(file, cols, rows) - if ret: - info_data[feature] = info_data.get(feature, {}) - info_data[feature]["layout"] = ret - except Exception as e: - _log_warning(info_data, f'led_config: {file.name}: {e}') + if feature: + # Process + for file in find_keyboard_c(keyboard): + try: + ret = find_led_config(file, cols, rows) + if ret: + info_data[feature] = info_data.get(feature, {}) + info_data[feature]["layout"] = ret + except Exception as e: + _log_warning(info_data, f'led_config: {file.name}: {e}') return info_data @@ -755,9 +745,6 @@ def arm_processor_rules(info_data, rules): info_data['processor_type'] = 'arm' info_data['protocol'] = 'ChibiOS' - if 'bootloader' not in info_data: - info_data['bootloader'] = 'unknown' - if 'STM32' in info_data['processor']: info_data['platform'] = 'STM32' elif 'MCU_SERIES' in rules: @@ -775,9 +762,6 @@ def avr_processor_rules(info_data, rules): info_data['platform'] = rules['ARCH'] if 'ARCH' in rules else 'unknown' info_data['protocol'] = 'V-USB' if rules.get('MCU') in VUSB_PROCESSORS else 'LUFA' - if 'bootloader' not in info_data: - info_data['bootloader'] = 'atmel-dfu' - # FIXME(fauxpark/anyone): Eventually we should detect the protocol by looking at PROTOCOL inherited from mcu_selection.mk: # info_data['protocol'] = 'V-USB' if rules.get('PROTOCOL') == 'VUSB' else 'LUFA' diff --git a/lib/python/qmk/keyboard.py b/lib/python/qmk/keyboard.py index 686d4fc403..6ddbba8fa5 100644 --- a/lib/python/qmk/keyboard.py +++ b/lib/python/qmk/keyboard.py @@ -69,7 +69,7 @@ def keyboard_folder(keyboard): This checks aliases and DEFAULT_FOLDER to resolve the actual path for a keyboard. """ - aliases = json_load(Path('data/mappings/keyboard_aliases.json')) + aliases = json_load(Path('data/mappings/keyboard_aliases.hjson')) if keyboard in aliases: keyboard = aliases[keyboard].get('target', keyboard) diff --git a/lib/python/qmk/keycodes.py b/lib/python/qmk/keycodes.py new file mode 100644 index 0000000000..cf1ee0767a --- /dev/null +++ b/lib/python/qmk/keycodes.py @@ -0,0 +1,57 @@ +from pathlib import Path + +from qmk.json_schema import deep_update, json_load, validate + +CONSTANTS_PATH = Path('data/constants/keycodes/') + + +def _validate(spec): + # first throw it to the jsonschema + validate(spec, 'qmk.keycodes.v1') + + # no duplicate keycodes + keycodes = [] + for value in spec['keycodes'].values(): + keycodes.append(value['key']) + keycodes.extend(value.get('aliases', [])) + duplicates = set([x for x in keycodes if keycodes.count(x) > 1]) + if duplicates: + raise ValueError(f'Keycode spec contains duplicate keycodes! ({",".join(duplicates)})') + + +def load_spec(version): + """Build keycode data from the requested spec file + """ + if version == 'latest': + version = list_versions()[0] + + file = CONSTANTS_PATH / f'keycodes_{version}.hjson' + if not file.exists(): + raise ValueError(f'Requested keycode spec ({version}) is invalid!') + + # Load base + spec = json_load(file) + + # Merge in fragments + fragments = CONSTANTS_PATH.glob(f'keycodes_{version}_*.hjson') + for file in fragments: + deep_update(spec, json_load(file)) + + # Sort? + spec['keycodes'] = dict(sorted(spec['keycodes'].items())) + + # Validate? + _validate(spec) + + return spec + + +def list_versions(): + """Return available versions - sorted newest first + """ + ret = [] + for file in CONSTANTS_PATH.glob('keycodes_[0-9].[0-9].[0-9].hjson'): + ret.append(file.stem.split('_')[1]) + + ret.sort(reverse=True) + return ret diff --git a/lib/python/qmk/keymap.py b/lib/python/qmk/keymap.py index f317f4d11e..315af35b73 100644 --- a/lib/python/qmk/keymap.py +++ b/lib/python/qmk/keymap.py @@ -266,7 +266,7 @@ def generate_c(keymap_json): new_macro = "".join(macro) new_macro = new_macro.replace('""', '') - macro_txt.append(f' case MACRO_{i}:') + macro_txt.append(f' case QK_MACRO_{i}:') macro_txt.append(f' SEND_STRING({new_macro});') macro_txt.append(' return false;') @@ -412,7 +412,7 @@ def list_keymaps(keyboard, c=True, json=True, additional_files=None, fullpath=Fa rules = rules_mk(keyboard) names = set() - if rules: + if rules is not None: keyboards_dir = Path('keyboards') kb_path = keyboards_dir / keyboard diff --git a/lib/python/qmk/tests/test_cli_commands.py b/lib/python/qmk/tests/test_cli_commands.py index fde8b079a3..e598b281a6 100644 --- a/lib/python/qmk/tests/test_cli_commands.py +++ b/lib/python/qmk/tests/test_cli_commands.py @@ -150,8 +150,8 @@ def test_json2c(): def test_json2c_macros(): result = check_subcommand("json2c", 'keyboards/handwired/pytest/macro/keymaps/default/keymap.json') check_returncode(result) - assert 'LAYOUT_ortho_1x1(MACRO_0)' in result.stdout - assert 'case MACRO_0:' in result.stdout + assert 'LAYOUT_ortho_1x1(QK_MACRO_0)' in result.stdout + assert 'case QK_MACRO_0:' in result.stdout assert 'SEND_STRING("Hello, World!"SS_TAP(X_ENTER));' in result.stdout @@ -263,16 +263,16 @@ def test_generate_rgb_breathe_table(): def test_generate_config_h(): result = check_subcommand('generate-config-h', '-kb', 'handwired/pytest/basic') check_returncode(result) - assert '# define DEVICE_VER 0x0001' in result.stdout - assert '# define DIODE_DIRECTION COL2ROW' in result.stdout - assert '# define MANUFACTURER none' in result.stdout - assert '# define PRODUCT pytest' in result.stdout - assert '# define PRODUCT_ID 0x6465' in result.stdout - assert '# define VENDOR_ID 0xFEED' in result.stdout - assert '# define MATRIX_COLS 1' in result.stdout - assert '# define MATRIX_COL_PINS { F4 }' in result.stdout - assert '# define MATRIX_ROWS 1' in result.stdout - assert '# define MATRIX_ROW_PINS { F5 }' in result.stdout + assert '# define DEVICE_VER 0x0001' in result.stdout + assert '# define DIODE_DIRECTION COL2ROW' in result.stdout + assert '# define MANUFACTURER "none"' in result.stdout + assert '# define PRODUCT "pytest"' in result.stdout + assert '# define PRODUCT_ID 0x6465' in result.stdout + assert '# define VENDOR_ID 0xFEED' in result.stdout + assert '# define MATRIX_COLS 1' in result.stdout + assert '# define MATRIX_COL_PINS { F4 }' in result.stdout + assert '# define MATRIX_ROWS 1' in result.stdout + assert '# define MATRIX_ROW_PINS { F5 }' in result.stdout def test_generate_rules_mk(): @@ -288,12 +288,6 @@ def test_generate_version_h(): assert '#define QMK_VERSION' in result.stdout -def test_generate_layouts(): - result = check_subcommand('generate-layouts', '-kb', 'handwired/pytest/basic') - check_returncode(result) - assert '#define LAYOUT_custom(k0A) {' in result.stdout - - def test_format_json_keyboard(): result = check_subcommand('format-json', '--format', 'keyboard', 'lib/python/qmk/tests/minimal_info.json') check_returncode(result) |