Create a system to map between info.json and config.h/rules.mk (#11548)

* generate rules.mk from a json mapping

* generate rules.mk from a json mapping

* support for config.h from json maps

* improve the mapping system

* document the mapping system

* move data/maps to data/mappings

* fix flake8 errors

* fixup LED_MATRIX_DRIVER

* remove product and description from the vision_division keymap level

* reduce the complexity of generate-rules-mk

* add tests for the generate commands

* fix qmk doctor when submodules are not clean

2 files changed, 97 insertions, 209 deletions

diff --git a/lib/python/qmk/cli/generate/config_h.py b/lib/python/qmk/cli/generate/config_h.py
index 1de84de7a9..7ddad745d1 100755
--- a/lib/python/qmk/cli/generate/config_h.py
+++ b/lib/python/qmk/cli/generate/config_h.py
@@ -1,62 +1,14 @@
 """Used by the make system to generate info_config.h from info.json.
 """
+from pathlib import Path
+
+from dotty_dict import dotty
 from milc import cli
 
-from qmk.constants import LED_INDICATORS
 from qmk.decorators import automagic_keyboard, automagic_keymap
-from qmk.info import info_json, rgblight_animations, rgblight_properties, rgblight_toggles
+from qmk.info import _json_load, info_json
 from qmk.path import is_keyboard, normpath
 
-usb_prop_map = {
-    'vid': 'VENDOR_ID',
-    'pid': 'PRODUCT_ID',
-    'device_ver': 'DEVICE_VER',
-}
-
-
-def debounce(debounce):
-    """Return the config.h lines that set debounce
-    """
-    return """
-#ifndef DEBOUNCE
-#   define DEBOUNCE %s
-#endif // DEBOUNCE
-""" % debounce
-
-
-def diode_direction(diode_direction):
-    """Return the config.h lines that set diode direction
-    """
-    return """
-#ifndef DIODE_DIRECTION
-#   define DIODE_DIRECTION %s
-#endif // DIODE_DIRECTION
-""" % diode_direction
-
-
-def keyboard_name(keyboard_name):
-    """Return the config.h lines that set the keyboard's name.
-    """
-    return """
-#ifndef DESCRIPTION
-#    define DESCRIPTION %s
-#endif // DESCRIPTION
-
-#ifndef PRODUCT
-#    define PRODUCT %s
-#endif // PRODUCT
-""" % (keyboard_name.replace("'", ""), keyboard_name.replace("'", ""))
-
-
-def manufacturer(manufacturer):
-    """Return the config.h lines that set the manufacturer.
-    """
-    return """
-#ifndef MANUFACTURER
-#    define MANUFACTURER %s
-#endif // MANUFACTURER
-""" % (manufacturer.replace("'", ""))
-
 
 def direct_pins(direct_pins):
     """Return the config.h lines that set the direct pins.
@@ -72,80 +24,34 @@ def direct_pins(direct_pins):
 
     return """
 #ifndef MATRIX_COLS
-#    define MATRIX_COLS %s
+#   define MATRIX_COLS %s
 #endif // MATRIX_COLS
 
 #ifndef MATRIX_ROWS
-#    define MATRIX_ROWS %s
+#   define MATRIX_ROWS %s
 #endif // MATRIX_ROWS
 
 #ifndef DIRECT_PINS
-#    define DIRECT_PINS {%s}
+#   define DIRECT_PINS {%s}
 #endif // DIRECT_PINS
 """ % (col_count, row_count, ','.join(rows))
 
 
-def col_pins(col_pins):
-    """Return the config.h lines that set the column pins.
-    """
-    cols = ','.join(map(str, [pin or 'NO_PIN' for pin in col_pins]))
-    col_num = len(col_pins)
-
-    return """
-#ifndef MATRIX_COLS
-#    define MATRIX_COLS %s
-#endif // MATRIX_COLS
-
-#ifndef MATRIX_COL_PINS
-#    define MATRIX_COL_PINS {%s}
-#endif // MATRIX_COL_PINS
-""" % (col_num, cols)
-
-
-def row_pins(row_pins):
-    """Return the config.h lines that set the row pins.
-    """
-    rows = ','.join(map(str, [pin or 'NO_PIN' for pin in row_pins]))
-    row_num = len(row_pins)
-
-    return """
-#ifndef MATRIX_ROWS
-#    define MATRIX_ROWS %s
-#endif // MATRIX_ROWS
-
-#ifndef MATRIX_ROW_PINS
-#    define MATRIX_ROW_PINS {%s}
-#endif // MATRIX_ROW_PINS
-""" % (row_num, rows)
-
-
-def indicators(config):
-    """Return the config.h lines that setup LED indicators.
+def pin_array(define, pins):
+    """Return the config.h lines that set a pin array.
     """
-    defines = []
+    pin_num = len(pins)
+    pin_array = ', '.join(map(str, [pin or 'NO_PIN' for pin in pins]))
 
-    for led, define in LED_INDICATORS.items():
-        if led in config:
-            defines.append('')
-            defines.append('#ifndef %s' % (define,))
-            defines.append('#   define %s %s' % (define, config[led]))
-            defines.append('#endif // %s' % (define,))
+    return f"""
+#ifndef {define}S
+#   define {define}S {pin_num}
+#endif // {define}S
 
-    return '\n'.join(defines)
-
-
-def layout_aliases(layout_aliases):
-    """Return the config.h lines that setup layout aliases.
-    """
-    defines = []
-
-    for alias, layout in layout_aliases.items():
-        defines.append('')
-        defines.append('#ifndef %s' % (alias,))
-        defines.append('#   define %s %s' % (alias, layout))
-        defines.append('#endif // %s' % (alias,))
-
-    return '\n'.join(defines)
+#ifndef {define}_PINS
+#   define {define}_PINS {{ {pin_array} }}
+#endif // {define}_PINS
+"""
 
 
 def matrix_pins(matrix_pins):
@@ -157,58 +63,14 @@ def matrix_pins(matrix_pins):
         pins.append(direct_pins(matrix_pins['direct']))
 
     if 'cols' in matrix_pins:
-        pins.append(col_pins(matrix_pins['cols']))
+        pins.append(pin_array('MATRIX_COL', matrix_pins['cols']))
 
     if 'rows' in matrix_pins:
-        pins.append(row_pins(matrix_pins['rows']))
+        pins.append(pin_array('MATRIX_ROW', matrix_pins['rows']))
 
     return '\n'.join(pins)
 
 
-def rgblight(config):
-    """Return the config.h lines that setup rgblight.
-    """
-    rgblight_config = []
-
-    for json_key, config_key in rgblight_properties.items():
-        if json_key in config:
-            rgblight_config.append('')
-            rgblight_config.append('#ifndef %s' % (config_key[0],))
-            rgblight_config.append('#   define %s %s' % (config_key[0], config[json_key]))
-            rgblight_config.append('#endif // %s' % (config_key[0],))
-
-    for json_key, config_key in rgblight_toggles.items():
-        if config.get(json_key):
-            rgblight_config.append('')
-            rgblight_config.append('#ifndef %s' % (config_key,))
-            rgblight_config.append('#   define %s' % (config_key,))
-            rgblight_config.append('#endif // %s' % (config_key,))
-
-    for json_key, config_key in rgblight_animations.items():
-        if 'animations' in config and config['animations'].get(json_key):
-            rgblight_config.append('')
-            rgblight_config.append('#ifndef %s' % (config_key,))
-            rgblight_config.append('#   define %s' % (config_key,))
-            rgblight_config.append('#endif // %s' % (config_key,))
-
-    return '\n'.join(rgblight_config)
-
-
-def usb_properties(usb_props):
-    """Return the config.h lines that setup USB params.
-    """
-    usb_lines = []
-
-    for info_name, config_name in usb_prop_map.items():
-        if info_name in usb_props:
-            usb_lines.append('')
-            usb_lines.append('#ifndef ' + config_name)
-            usb_lines.append('#    define %s %s' % (config_name, usb_props[info_name]))
-            usb_lines.append('#endif // ' + config_name)
-
-    return '\n'.join(usb_lines)
-
-
 @cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to')
 @cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages")
 @cli.argument('-kb', '--keyboard', help='Keyboard to generate config.h for.')
@@ -228,39 +90,52 @@ def generate_config_h(cli):
         cli.log.error('Invalid keyboard: "%s"', cli.config.generate_config_h.keyboard)
         return False
 
-    # Build the info.json file
-    kb_info_json = info_json(cli.config.generate_config_h.keyboard)
-
     # Build the info_config.h file.
-    config_h_lines = ['/* This file was generated by `qmk generate-config-h`. Do not edit or copy.' ' */', '', '#pragma once']
+    kb_info_json = dotty(info_json(cli.config.generate_config_h.keyboard))
+    info_config_map = _json_load(Path('data/mappings/info_config.json'))
 
-    if 'debounce' in kb_info_json:
-        config_h_lines.append(debounce(kb_info_json['debounce']))
-
-    if 'diode_direction' in kb_info_json:
-        config_h_lines.append(diode_direction(kb_info_json['diode_direction']))
-
-    if 'indicators' in kb_info_json:
-        config_h_lines.append(indicators(kb_info_json['indicators']))
-
-    if 'keyboard_name' in kb_info_json:
-        config_h_lines.append(keyboard_name(kb_info_json['keyboard_name']))
-
-    if 'layout_aliases' in kb_info_json:
-        config_h_lines.append(layout_aliases(kb_info_json['layout_aliases']))
-
-    if 'manufacturer' in kb_info_json:
-        config_h_lines.append(manufacturer(kb_info_json['manufacturer']))
+    config_h_lines = ['/* This file was generated by `qmk generate-config-h`. Do not edit or copy.' ' */', '', '#pragma once']
 
-    if 'rgblight' in kb_info_json:
-        config_h_lines.append(rgblight(kb_info_json['rgblight']))
+    # Iterate through the info_config map to generate basic things
+    for config_key, info_dict in info_config_map.items():
+        info_key = info_dict['info_key']
+        key_type = info_dict.get('value_type', 'str')
+        to_config = info_dict.get('to_config', True)
+
+        if not to_config:
+            continue
+
+        try:
+            config_value = kb_info_json[info_key]
+        except KeyError:
+            continue
+
+        if key_type.startswith('array'):
+            config_h_lines.append('')
+            config_h_lines.append(f'#ifndef {config_key}')
+            config_h_lines.append(f'#   define {config_key} {{ {", ".join(map(str, config_value))} }}')
+            config_h_lines.append(f'#endif // {config_key}')
+        elif key_type == 'bool':
+            if config_value:
+                config_h_lines.append('')
+                config_h_lines.append(f'#ifndef {config_key}')
+                config_h_lines.append(f'#   define {config_key}')
+                config_h_lines.append(f'#endif // {config_key}')
+        elif key_type == 'mapping':
+            for key, value in config_value.items():
+                config_h_lines.append('')
+                config_h_lines.append(f'#ifndef {key}')
+                config_h_lines.append(f'#   define {key} {value}')
+                config_h_lines.append(f'#endif // {key}')
+        else:
+            config_h_lines.append('')
+            config_h_lines.append(f'#ifndef {config_key}')
+            config_h_lines.append(f'#   define {config_key} {config_value}')
+            config_h_lines.append(f'#endif // {config_key}')
 
     if 'matrix_pins' in kb_info_json:
         config_h_lines.append(matrix_pins(kb_info_json['matrix_pins']))
 
-    if 'usb' in kb_info_json:
-        config_h_lines.append(usb_properties(kb_info_json['usb']))
-
     # Show the results
     config_h = '\n'.join(config_h_lines)
 
diff --git a/lib/python/qmk/cli/generate/rules_mk.py b/lib/python/qmk/cli/generate/rules_mk.py
index b262e3c666..af740f341d 100755
--- a/lib/python/qmk/cli/generate/rules_mk.py
+++ b/lib/python/qmk/cli/generate/rules_mk.py
@@ -1,16 +1,37 @@
 """Used by the make system to generate a rules.mk
 """
+from pathlib import Path
+
+from dotty_dict import dotty
 from milc import cli
 
 from qmk.decorators import automagic_keyboard, automagic_keymap
-from qmk.info import info_json
+from qmk.info import _json_load, info_json
 from qmk.path import is_keyboard, normpath
 
-info_to_rules = {
-    'board': 'BOARD',
-    'bootloader': 'BOOTLOADER',
-    'processor': 'MCU',
-}
+
+def process_mapping_rule(kb_info_json, rules_key, info_dict):
+    """Return the rules.mk line(s) for a mapping rule.
+    """
+    if not info_dict.get('to_c', True):
+        return None
+
+    info_key = info_dict['info_key']
+    key_type = info_dict.get('value_type', 'str')
+
+    try:
+        rules_value = kb_info_json[info_key]
+    except KeyError:
+        return None
+
+    if key_type == 'array':
+        return f'{rules_key} ?= {" ".join(rules_value)}'
+    elif key_type == 'bool':
+        return f'{rules_key} ?= {"on" if rules_value else "off"}'
+    elif key_type == 'mapping':
+        return '\n'.join([f'{key} ?= {value}' for key, value in rules_value.items()])
+
+    return f'{rules_key} ?= {rules_value}'
 
 
 @cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to')
@@ -22,7 +43,6 @@ info_to_rules = {
 def generate_rules_mk(cli):
     """Generates a rules.mk file from info.json.
     """
-    # Determine our keyboard(s)
     if not cli.config.generate_rules_mk.keyboard:
         cli.log.error('Missing paramater: --keyboard')
         cli.subcommands['info'].print_help()
@@ -32,16 +52,18 @@ def generate_rules_mk(cli):
         cli.log.error('Invalid keyboard: "%s"', cli.config.generate_rules_mk.keyboard)
         return False
 
-    # Build the info.json file
-    kb_info_json = info_json(cli.config.generate_rules_mk.keyboard)
+    kb_info_json = dotty(info_json(cli.config.generate_rules_mk.keyboard))
+    info_rules_map = _json_load(Path('data/mappings/info_rules.json'))
     rules_mk_lines = ['# This file was generated by `qmk generate-rules-mk`. Do not edit or copy.', '']
 
-    # Bring in settings
-    for info_key, rule_key in info_to_rules.items():
-        if info_key in kb_info_json:
-            rules_mk_lines.append(f'{rule_key} ?= {kb_info_json[info_key]}')
+    # Iterate through the info_rules map to generate basic rules
+    for rules_key, info_dict in info_rules_map.items():
+        new_entry = process_mapping_rule(kb_info_json, rules_key, info_dict)
+
+        if new_entry:
+            rules_mk_lines.append(new_entry)
 
-    # Find features that should be enabled
+    # Iterate through features to enable/disable them
     if 'features' in kb_info_json:
         for feature, enabled in kb_info_json['features'].items():
             if feature == 'bootmagic_lite' and enabled:
@@ -51,15 +73,6 @@ def generate_rules_mk(cli):
                 enabled = 'yes' if enabled else 'no'
                 rules_mk_lines.append(f'{feature}_ENABLE ?= {enabled}')
 
-    # Set the LED driver
-    if 'led_matrix' in kb_info_json and 'driver' in kb_info_json['led_matrix']:
-        driver = kb_info_json['led_matrix']['driver']
-        rules_mk_lines.append(f'LED_MATRIX_DRIVER ?= {driver}')
-
-    # Add community layouts
-    if 'community_layouts' in kb_info_json:
-        rules_mk_lines.append(f'LAYOUTS ?= {" ".join(kb_info_json["community_layouts"])}')
-
     # Show the results
     rules_mk = '\n'.join(rules_mk_lines) + '\n'
 
@@ -72,7 +85,7 @@ def generate_rules_mk(cli):
         if cli.args.quiet:
             print(cli.args.output)
         else:
-            cli.log.info('Wrote info_config.h to %s.', cli.args.output)
+            cli.log.info('Wrote rules.mk to %s.', cli.args.output)
 
     else:
         print(rules_mk)