From 47b9b110097a864d6ab76516b2213afd59948527 Mon Sep 17 00:00:00 2001 From: Zach White Date: Wed, 30 Dec 2020 10:27:37 -0800 Subject: Configure keyboard matrix from info.json (#10817) * Make parameters from info.json available to the build system * move all clueboard settings to info.json * code formatting * make flake8 happy * make flake8 happy * make qmk lint happy * Add support for specifying led indicators in json * move led indicators to the clueboard info.json * Apply suggestions from code review Co-authored-by: Erovia * add missing docstring Co-authored-by: Erovia --- lib/python/qmk/info.py | 355 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 296 insertions(+), 59 deletions(-) (limited to 'lib/python/qmk/info.py') diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py index f476dc666d..d7b128aa66 100644 --- a/lib/python/qmk/info.py +++ b/lib/python/qmk/info.py @@ -6,13 +6,45 @@ from pathlib import Path from milc import cli -from qmk.constants import CHIBIOS_PROCESSORS, LUFA_PROCESSORS, VUSB_PROCESSORS +from qmk.constants import CHIBIOS_PROCESSORS, LUFA_PROCESSORS, VUSB_PROCESSORS, LED_INDICATORS from qmk.c_parse import find_layouts from qmk.keyboard import config_h, rules_mk from qmk.keymap import list_keymaps from qmk.makefile import parse_rules_mk_file from qmk.math import compute +rgblight_properties = { + 'led_count': 'RGBLED_NUM', + 'pin': 'RGB_DI_PIN', + 'split_count': 'RGBLED_SPLIT', + 'max_brightness': 'RGBLIGHT_LIMIT_VAL', + 'hue_steps': 'RGBLIGHT_HUE_STEP', + 'saturation_steps': 'RGBLIGHT_SAT_STEP', + 'brightness_steps': 'RGBLIGHT_VAL_STEP' +} + +rgblight_toggles = { + 'sleep': 'RGBLIGHT_SLEEP', + 'split': 'RGBLIGHT_SPLIT', +} + +rgblight_animations = { + 'all': 'RGBLIGHT_ANIMATIONS', + 'alternating': 'RGBLIGHT_EFFECT_ALTERNATING', + 'breathing': 'RGBLIGHT_EFFECT_BREATHING', + 'christmas': 'RGBLIGHT_EFFECT_CHRISTMAS', + 'knight': 'RGBLIGHT_EFFECT_KNIGHT', + 'rainbow_mood': 'RGBLIGHT_EFFECT_RAINBOW_MOOD', + 'rainbow_swirl': 'RGBLIGHT_EFFECT_RAINBOW_SWIRL', + 'rgb_test': 'RGBLIGHT_EFFECT_RGB_TEST', + 'snake': 'RGBLIGHT_EFFECT_SNAKE', + 'static_gradient': 'RGBLIGHT_EFFECT_STATIC_GRADIENT', + 'twinkle': 'RGBLIGHT_EFFECT_TWINKLE' +} + +true_values = ['1', 'on', 'yes'] +false_values = ['0', 'off', 'no'] + def info_json(keyboard): """Generate the info.json data for a specific keyboard. @@ -38,8 +70,9 @@ def info_json(keyboard): info_data['keymaps'][keymap.name] = {'url': f'https://raw.githubusercontent.com/qmk/qmk_firmware/master/{keymap}/keymap.json'} # Populate layout data - for layout_name, layout_json in _find_all_layouts(info_data, keyboard, rules).items(): + for layout_name, layout_json in _find_all_layouts(info_data, keyboard).items(): if not layout_name.startswith('LAYOUT_kc'): + layout_json['c_macro'] = True info_data['layouts'][layout_name] = layout_json # Merge in the data from info.json, config.h, and rules.mk @@ -47,34 +80,179 @@ def info_json(keyboard): info_data = _extract_config_h(info_data) info_data = _extract_rules_mk(info_data) + # Make sure we have at least one layout + if not info_data.get('layouts'): + _log_error(info_data, 'No LAYOUTs defined! Need at least one layout defined in the keyboard.h or info.json.') + + # Make sure we supply layout macros for the community layouts we claim to support + # FIXME(skullydazed): This should be populated into info.json and read from there instead + if 'LAYOUTS' in rules and info_data.get('layouts'): + # Match these up against the supplied layouts + supported_layouts = rules['LAYOUTS'].strip().split() + for layout_name in sorted(info_data['layouts']): + layout_name = layout_name[7:] + + if layout_name in supported_layouts: + supported_layouts.remove(layout_name) + + if supported_layouts: + for supported_layout in supported_layouts: + _log_error(info_data, 'Claims to support community layout %s but no LAYOUT_%s() macro found' % (supported_layout, supported_layout)) + return info_data -def _extract_config_h(info_data): - """Pull some keyboard information from existing rules.mk files +def _extract_debounce(info_data, config_c): + """Handle debounce. + """ + if 'debounce' in info_data and 'DEBOUNCE' in config_c: + _log_warning(info_data, 'Debounce is specified in both info.json and config.h, the config.h value wins.') + + if 'DEBOUNCE' in config_c: + info_data['debounce'] = config_c.get('DEBOUNCE') + + return info_data + + +def _extract_diode_direction(info_data, config_c): + """Handle the diode direction. + """ + if 'diode_direction' in info_data and 'DIODE_DIRECTION' in config_c: + _log_warning(info_data, 'Diode direction is specified in both info.json and config.h, the config.h value wins.') + + if 'DIODE_DIRECTION' in config_c: + info_data['diode_direction'] = config_c.get('DIODE_DIRECTION') + + return info_data + + +def _extract_indicators(info_data, config_c): + """Find the LED indicator information. + """ + for json_key, config_key in LED_INDICATORS.items(): + if json_key in info_data.get('indicators', []) and config_key in config_c: + _log_warning(info_data, f'Indicator {json_key} is specified in both info.json and config.h, the config.h value wins.') + + if config_key in config_c: + info_data['indicators'][json_key] = config_c.get(config_key) + + return info_data + + +def _extract_community_layouts(info_data, rules): + """Find the community layouts in rules.mk. + """ + community_layouts = rules['LAYOUTS'].split() if 'LAYOUTS' in rules else [] + + if 'community_layouts' in info_data: + for layout in community_layouts: + if layout not in info_data['community_layouts']: + community_layouts.append(layout) + + else: + info_data['community_layouts'] = community_layouts + + return info_data + + +def _extract_features(info_data, rules): + """Find all the features enabled in rules.mk. + """ + for key, value in rules.items(): + if key.endswith('_ENABLE'): + key = '_'.join(key.split('_')[:-1]).lower() + value = True if value in true_values else False if value in false_values else value + + if 'config_h_features' not in info_data: + info_data['config_h_features'] = {} + + if 'features' not in info_data: + info_data['features'] = {} + + if key in info_data['features']: + _log_warning(info_data, 'Feature %s is specified in both info.json and rules.mk, the rules.mk value wins.' % (key,)) + + info_data['features'][key] = value + info_data['config_h_features'][key] = value + + return info_data + + +def _extract_rgblight(info_data, config_c): + """Handle the rgblight configuration + """ + rgblight = info_data.get('rgblight', {}) + animations = rgblight.get('animations', {}) + + for json_key, config_key in rgblight_properties.items(): + if config_key in config_c: + if json_key in rgblight: + _log_warning(info_data, 'RGB Light: %s is specified in both info.json and config.h, the config.h value wins.' % (json_key,)) + + rgblight[json_key] = config_c[config_key] + + for json_key, config_key in rgblight_toggles.items(): + if config_key in config_c: + if json_key in rgblight: + _log_warning(info_data, 'RGB Light: %s is specified in both info.json and config.h, the config.h value wins.', json_key) + + rgblight[json_key] = config_c[config_key] + + for json_key, config_key in rgblight_animations.items(): + if config_key in config_c: + if json_key in animations: + _log_warning(info_data, 'RGB Light: animations: %s is specified in both info.json and config.h, the config.h value wins.' % (json_key,)) + + animations[json_key] = config_c[config_key] + + if animations: + rgblight['animations'] = animations + + if rgblight: + info_data['rgblight'] = rgblight + + return info_data + + +def _extract_matrix_info(info_data, config_c): + """Populate the matrix information. """ - config_c = config_h(info_data['keyboard_folder']) row_pins = config_c.get('MATRIX_ROW_PINS', '').replace('{', '').replace('}', '').strip() col_pins = config_c.get('MATRIX_COL_PINS', '').replace('{', '').replace('}', '').strip() direct_pins = config_c.get('DIRECT_PINS', '').replace(' ', '')[1:-1] - info_data['diode_direction'] = config_c.get('DIODE_DIRECTION') - info_data['matrix_size'] = { - 'rows': compute(config_c.get('MATRIX_ROWS', '0')), - 'cols': compute(config_c.get('MATRIX_COLS', '0')), - } - info_data['matrix_pins'] = {} + if 'MATRIX_ROWS' in config_c and 'MATRIX_COLS' in config_c: + if 'matrix_size' in info_data: + _log_warning(info_data, 'Matrix size is specified in both info.json and config.h, the config.h values win.') - if row_pins: - info_data['matrix_pins']['rows'] = row_pins.split(',') - if col_pins: - info_data['matrix_pins']['cols'] = col_pins.split(',') + info_data['matrix_size'] = { + 'rows': compute(config_c.get('MATRIX_ROWS', '0')), + 'cols': compute(config_c.get('MATRIX_COLS', '0')), + } + + if row_pins and col_pins: + if 'matrix_pins' in info_data: + _log_warning(info_data, 'Matrix pins are specified in both info.json and config.h, the config.h values win.') + + info_data['matrix_pins'] = {} + + if row_pins: + info_data['matrix_pins']['rows'] = row_pins.split(',') + + if col_pins: + info_data['matrix_pins']['cols'] = col_pins.split(',') if direct_pins: + if 'matrix_pins' in info_data: + _log_warning(info_data, 'Direct pins are specified in both info.json and config.h, the config.h values win.') + + info_data['matrix_pins'] = {} direct_pin_array = [] + for row in direct_pins.split('},{'): if row.startswith('{'): row = row[1:] + if row.endswith('}'): row = row[:-1] @@ -86,15 +264,43 @@ def _extract_config_h(info_data): direct_pin_array[-1].append(pin) - info_data['matrix_pins']['direct'] = direct_pin_array + info_data['matrix_pins']['direct'] = direct_pin_array - info_data['usb'] = { - 'vid': config_c.get('VENDOR_ID'), - 'pid': config_c.get('PRODUCT_ID'), - 'device_ver': config_c.get('DEVICE_VER'), - 'manufacturer': config_c.get('MANUFACTURER'), - 'product': config_c.get('PRODUCT'), - } + return info_data + + +def _extract_usb_info(info_data, config_c): + """Populate the USB information. + """ + usb_properties = {'vid': 'VENDOR_ID', 'pid': 'PRODUCT_ID', 'device_ver': 'DEVICE_VER'} + + if 'usb' not in info_data: + info_data['usb'] = {} + + for info_name, config_name in usb_properties.items(): + if config_name in config_c: + if info_name in info_data['usb']: + _log_warning(info_data, '%s in config.h is overwriting usb.%s in info.json' % (config_name, info_name)) + + info_data['usb'][info_name] = config_c[config_name] + + elif info_name not in info_data['usb']: + _log_error(info_data, '%s not specified in config.h, and %s not specified in info.json. One is required.' % (config_name, info_name)) + + return info_data + + +def _extract_config_h(info_data): + """Pull some keyboard information from existing config.h files + """ + config_c = config_h(info_data['keyboard_folder']) + + _extract_debounce(info_data, config_c) + _extract_diode_direction(info_data, config_c) + _extract_indicators(info_data, config_c) + _extract_matrix_info(info_data, config_c) + _extract_usb_info(info_data, config_c) + _extract_rgblight(info_data, config_c) return info_data @@ -106,16 +312,52 @@ def _extract_rules_mk(info_data): mcu = rules.get('MCU') if mcu in CHIBIOS_PROCESSORS: - return arm_processor_rules(info_data, rules) + arm_processor_rules(info_data, rules) elif mcu in LUFA_PROCESSORS + VUSB_PROCESSORS: - return avr_processor_rules(info_data, rules) + avr_processor_rules(info_data, rules) + + else: + cli.log.warning("%s: Unknown MCU: %s" % (info_data['keyboard_folder'], mcu)) + unknown_processor_rules(info_data, rules) + + _extract_community_layouts(info_data, rules) + _extract_features(info_data, rules) + + return info_data - msg = "Unknown MCU: " + str(mcu) - _log_warning(info_data, msg) +def _merge_layouts(info_data, new_info_data): + """Merge new_info_data into info_data in an intelligent way. + """ + for layout_name, layout_json in new_info_data['layouts'].items(): + if layout_name in info_data['layouts']: + # Pull in layouts we have a macro for + if len(info_data['layouts'][layout_name]['layout']) != len(layout_json['layout']): + msg = '%s: %s: Number of elements in info.json does not match! info.json:%s != %s:%s' + _log_error(info_data, msg % (info_data['keyboard_folder'], layout_name, len(layout_json['layout']), layout_name, len(info_data['layouts'][layout_name]['layout']))) + else: + for i, key in enumerate(info_data['layouts'][layout_name]['layout']): + key.update(layout_json['layout'][i]) + else: + # Pull in layouts that have matrix data + missing_matrix = False + for key in layout_json['layout']: + if 'matrix' not in key: + missing_matrix = True + + if not missing_matrix: + if layout_name in info_data['layouts']: + # Update an existing layout with new data + for i, key in enumerate(info_data['layouts'][layout_name]['layout']): + key.update(layout_json['layout'][i]) - return unknown_processor_rules(info_data, rules) + else: + # Copy in the new layout wholesale + layout_json['c_macro'] = False + info_data['layouts'][layout_name] = layout_json + + return info_data def _search_keyboard_h(path): @@ -131,34 +373,21 @@ def _search_keyboard_h(path): return layouts -def _find_all_layouts(info_data, keyboard, rules): +def _find_all_layouts(info_data, keyboard): """Looks for layout macros associated with this keyboard. """ layouts = _search_keyboard_h(Path(keyboard)) if not layouts: - # If we didn't find any layouts above we widen our search. This is error - # prone which is why we want to encourage people to follow the standard above. - _log_warning(info_data, 'Falling back to searching for KEYMAP/LAYOUT macros.') + # If we don't find any layouts from info.json or keyboard.h we widen our search. This is error prone which is why we want to encourage people to follow the standard above. + info_data['parse_warnings'].append('%s: Falling back to searching for KEYMAP/LAYOUT macros.' % (keyboard)) + for file in glob('keyboards/%s/*.h' % keyboard): if file.endswith('.h'): these_layouts = find_layouts(file) if these_layouts: layouts.update(these_layouts) - if 'LAYOUTS' in rules: - # Match these up against the supplied layouts - supported_layouts = rules['LAYOUTS'].strip().split() - for layout_name in sorted(layouts): - if not layout_name.startswith('LAYOUT_'): - continue - layout_name = layout_name[7:] - if layout_name in supported_layouts: - supported_layouts.remove(layout_name) - - if supported_layouts: - _log_error(info_data, 'Missing LAYOUT() macro for %s' % (', '.join(supported_layouts))) - return layouts @@ -231,32 +460,40 @@ def merge_info_jsons(keyboard, info_data): for info_file in find_info_json(keyboard): # Load and validate the JSON data try: - with info_file.open('r') as info_fd: - new_info_data = json.load(info_fd) + new_info_data = json.load(info_file.open('r')) except Exception as e: _log_error(info_data, "Invalid JSON in file %s: %s: %s" % (str(info_file), e.__class__.__name__, e)) - continue + new_info_data = {} if not isinstance(new_info_data, dict): _log_error(info_data, "Invalid file %s, root object should be a dictionary." % (str(info_file),)) continue # Copy whitelisted keys into `info_data` - for key in ('keyboard_name', 'manufacturer', 'identifier', 'url', 'maintainer', 'processor', 'bootloader', 'width', 'height'): + for key in ('debounce', 'diode_direction', 'indicators', 'keyboard_name', 'manufacturer', 'identifier', 'url', 'maintainer', 'processor', 'bootloader', 'width', 'height'): if key in new_info_data: info_data[key] = new_info_data[key] - # Merge the layouts in + # Deep merge certain keys + # FIXME(skullydazed/anyone): this should be generalized more so that we can inteligently merge more than one level deep. It would be nice if we could filter on valid keys too. That may have to wait for a future where we use openapi or something. + for key in ('features', 'layout_aliases', 'matrix_pins', 'rgblight', 'usb'): + if key in new_info_data: + if key not in info_data: + info_data[key] = {} + + info_data[key].update(new_info_data[key]) + + # Merge the layouts + if 'community_layouts' in new_info_data: + if 'community_layouts' in info_data: + for layout in new_info_data['community_layouts']: + if layout not in info_data['community_layouts']: + info_data['community_layouts'].append(layout) + else: + info_data['community_layouts'] = new_info_data['community_layouts'] + if 'layouts' in new_info_data: - for layout_name, json_layout in new_info_data['layouts'].items(): - # Only pull in layouts we have a macro for - if layout_name in info_data['layouts']: - if info_data['layouts'][layout_name]['key_count'] != len(json_layout['layout']): - msg = '%s: Number of elements in info.json does not match! info.json:%s != %s:%s' - _log_error(info_data, msg % (layout_name, len(json_layout['layout']), layout_name, len(info_data['layouts'][layout_name]['layout']))) - else: - for i, key in enumerate(info_data['layouts'][layout_name]['layout']): - key.update(json_layout['layout'][i]) + _merge_layouts(info_data, new_info_data) return info_data -- cgit v1.2.3 From 11bd98f684148ed9577b263189121e52027d66d9 Mon Sep 17 00:00:00 2001 From: Zach White Date: Sat, 2 Jan 2021 18:08:17 -0800 Subject: Fix broken keyboards (#11412) * Fix a couple errors * add a dependency for the generated headers --- lib/python/qmk/info.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib/python/qmk/info.py') diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py index d7b128aa66..2954a17e09 100644 --- a/lib/python/qmk/info.py +++ b/lib/python/qmk/info.py @@ -134,6 +134,9 @@ def _extract_indicators(info_data, config_c): _log_warning(info_data, f'Indicator {json_key} is specified in both info.json and config.h, the config.h value wins.') if config_key in config_c: + if 'indicators' not in info_data: + info_data['indicators'] = {} + info_data['indicators'][json_key] = config_c.get(config_key) return info_data -- cgit v1.2.3 From 6daa204363dfaa7f31d935a9913d6b35ec22e971 Mon Sep 17 00:00:00 2001 From: Zach White Date: Sun, 3 Jan 2021 15:45:06 -0800 Subject: fix keyboards with apostrophes in their names --- lib/python/qmk/info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/python/qmk/info.py') diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py index 2954a17e09..4611874e85 100644 --- a/lib/python/qmk/info.py +++ b/lib/python/qmk/info.py @@ -345,7 +345,7 @@ def _merge_layouts(info_data, new_info_data): else: # Pull in layouts that have matrix data missing_matrix = False - for key in layout_json['layout']: + for key in layout_json.get('layout', {}): if 'matrix' not in key: missing_matrix = True -- cgit v1.2.3 From ededff8556daff544633cb143cb6d939afd09014 Mon Sep 17 00:00:00 2001 From: Zach White Date: Tue, 1 Dec 2020 12:52:02 -0800 Subject: validate keyboard data with jsonschema --- lib/python/qmk/info.py | 151 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 140 insertions(+), 11 deletions(-) (limited to 'lib/python/qmk/info.py') diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py index 4611874e85..1cf12190d6 100644 --- a/lib/python/qmk/info.py +++ b/lib/python/qmk/info.py @@ -4,6 +4,7 @@ import json from glob import glob from pathlib import Path +import jsonschema from milc import cli from qmk.constants import CHIBIOS_PROCESSORS, LUFA_PROCESSORS, VUSB_PROCESSORS, LED_INDICATORS @@ -13,6 +14,17 @@ from qmk.keymap import list_keymaps from qmk.makefile import parse_rules_mk_file from qmk.math import compute +led_matrix_properties = { + 'driver_count': 'LED_DRIVER_COUNT', + 'driver_addr1': 'LED_DRIVER_ADDR_1', + 'driver_addr2': 'LED_DRIVER_ADDR_2', + 'driver_addr3': 'LED_DRIVER_ADDR_3', + 'driver_addr4': 'LED_DRIVER_ADDR_4', + 'led_count': 'LED_DRIVER_LED_COUNT', + 'timeout': 'ISSI_TIMEOUT', + 'persistence': 'ISSI_PERSISTENCE' +} + rgblight_properties = { 'led_count': 'RGBLED_NUM', 'pin': 'RGB_DI_PIN', @@ -80,6 +92,15 @@ def info_json(keyboard): info_data = _extract_config_h(info_data) info_data = _extract_rules_mk(info_data) + # Validate against the jsonschema + try: + keyboard_api_validate(info_data) + + except jsonschema.ValidationError as e: + cli.log.error('Invalid info.json data: %s', e.message) + print(dir(e)) + exit() + # Make sure we have at least one layout if not info_data.get('layouts'): _log_error(info_data, 'No LAYOUTs defined! Need at least one layout defined in the keyboard.h or info.json.') @@ -102,6 +123,50 @@ def info_json(keyboard): return info_data +def _json_load(json_file): + """Load a json file from disk. + + Note: file must be a Path object. + """ + try: + return json.load(json_file.open()) + + except json.decoder.JSONDecodeError as e: + cli.log.error('Invalid JSON encountered attempting to load {fg_cyan}%s{fg_reset}:\n\t{fg_red}%s', json_file, e) + exit(1) + + +def _jsonschema(schema_name): + """Read a jsonschema file from disk. + """ + schema_path = Path(f'data/schemas/{schema_name}.jsonschema') + + if not schema_path.exists(): + schema_path = Path('data/schemas/false.jsonschema') + + return _json_load(schema_path) + + +def keyboard_validate(data): + """Validates data against the keyboard jsonschema. + """ + schema = _jsonschema('keyboard') + validator = jsonschema.Draft7Validator(schema).validate + + return validator(data) + + +def keyboard_api_validate(data): + """Validates data against the api_keyboard jsonschema. + """ + base = _jsonschema('keyboard') + relative = _jsonschema('api_keyboard') + resolver = jsonschema.RefResolver.from_schema(base) + validator = jsonschema.Draft7Validator(relative, resolver=resolver).validate + + return validator(data) + + def _extract_debounce(info_data, config_c): """Handle debounce. """ @@ -109,7 +174,7 @@ def _extract_debounce(info_data, config_c): _log_warning(info_data, 'Debounce is specified in both info.json and config.h, the config.h value wins.') if 'DEBOUNCE' in config_c: - info_data['debounce'] = config_c.get('DEBOUNCE') + info_data['debounce'] = int(config_c['DEBOUNCE']) return info_data @@ -181,8 +246,36 @@ def _extract_features(info_data, rules): return info_data +def _extract_led_drivers(info_data, rules): + """Find all the LED drivers set in rules.mk. + """ + if 'LED_MATRIX_DRIVER' in rules: + if 'led_matrix' not in info_data: + info_data['led_matrix'] = {} + + if info_data['led_matrix'].get('driver'): + _log_warning(info_data, 'LED Matrix driver is specified in both info.json and rules.mk, the rules.mk value wins.') + + info_data['led_matrix']['driver'] = rules['LED_MATRIX_DRIVER'] + + return info_data + + +def _extract_led_matrix(info_data, config_c): + """Handle the led_matrix configuration. + """ + led_matrix = info_data.get('led_matrix', {}) + + for json_key, config_key in led_matrix_properties.items(): + if config_key in config_c: + if json_key in led_matrix: + _log_warning(info_data, 'LED Matrix: %s is specified in both info.json and config.h, the config.h value wins.' % (json_key,)) + + led_matrix[json_key] = config_c[config_key] + + def _extract_rgblight(info_data, config_c): - """Handle the rgblight configuration + """Handle the rgblight configuration. """ rgblight = info_data.get('rgblight', {}) animations = rgblight.get('animations', {}) @@ -303,6 +396,7 @@ def _extract_config_h(info_data): _extract_indicators(info_data, config_c) _extract_matrix_info(info_data, config_c) _extract_usb_info(info_data, config_c) + _extract_led_matrix(info_data, config_c) _extract_rgblight(info_data, config_c) return info_data @@ -326,6 +420,7 @@ def _extract_rules_mk(info_data): _extract_community_layouts(info_data, rules) _extract_features(info_data, rules) + _extract_led_drivers(info_data, rules) return info_data @@ -412,13 +507,28 @@ def arm_processor_rules(info_data, rules): """Setup the default info for an ARM board. """ info_data['processor_type'] = 'arm' - info_data['bootloader'] = rules['BOOTLOADER'] if 'BOOTLOADER' in rules else 'unknown' - info_data['processor'] = rules['MCU'] if 'MCU' in rules else 'unknown' info_data['protocol'] = 'ChibiOS' - if info_data['bootloader'] == 'unknown': + if 'MCU' in rules: + if 'processor' in info_data: + _log_warning(info_data, 'Processor/MCU is specified in both info.json and rules.mk, the rules.mk value wins.') + + info_data['processor'] = rules['MCU'] + + elif 'processor' not in info_data: + info_data['processor'] = 'unknown' + + if 'BOOTLOADER' in rules: + if 'bootloader' in info_data: + _log_warning(info_data, 'Bootloader is specified in both info.json and rules.mk, the rules.mk value wins.') + + info_data['bootloader'] = rules['BOOTLOADER'] + + else: if 'STM32' in info_data['processor']: info_data['bootloader'] = 'stm32-dfu' + else: + info_data['bootloader'] = 'unknown' if 'STM32' in info_data['processor']: info_data['platform'] = 'STM32' @@ -436,9 +546,25 @@ def avr_processor_rules(info_data, rules): info_data['processor_type'] = 'avr' info_data['bootloader'] = rules['BOOTLOADER'] if 'BOOTLOADER' in rules else 'atmel-dfu' info_data['platform'] = rules['ARCH'] if 'ARCH' in rules else 'unknown' - info_data['processor'] = rules['MCU'] if 'MCU' in rules else 'unknown' info_data['protocol'] = 'V-USB' if rules.get('MCU') in VUSB_PROCESSORS else 'LUFA' + if 'MCU' in rules: + if 'processor' in info_data: + _log_warning(info_data, 'Processor/MCU is specified in both info.json and rules.mk, the rules.mk value wins.') + + info_data['processor'] = rules['MCU'] + + elif 'processor' not in info_data: + info_data['processor'] = 'unknown' + + if 'BOOTLOADER' in rules: + if 'bootloader' in info_data: + _log_warning(info_data, 'Bootloader is specified in both info.json and rules.mk, the rules.mk value wins.') + + info_data['bootloader'] = rules['BOOTLOADER'] + else: + info_data['bootloader'] = 'atmel-dfu' + # FIXME(fauxpark/anyone): Eventually we should detect the protocol by looking at PROTOCOL inherited from mcu_selection.mk: # info_data['protocol'] = 'V-USB' if rules.get('PROTOCOL') == 'VUSB' else 'LUFA' @@ -463,10 +589,13 @@ def merge_info_jsons(keyboard, info_data): for info_file in find_info_json(keyboard): # Load and validate the JSON data try: - new_info_data = json.load(info_file.open('r')) - except Exception as e: - _log_error(info_data, "Invalid JSON in file %s: %s: %s" % (str(info_file), e.__class__.__name__, e)) - new_info_data = {} + new_info_data = _json_load(info_file) + keyboard_validate(new_info_data) + + except jsonschema.ValidationError as e: + cli.log.error('Invalid info.json data: %s', e.message) + cli.log.error('Not including file %s', info_file) + continue if not isinstance(new_info_data, dict): _log_error(info_data, "Invalid file %s, root object should be a dictionary." % (str(info_file),)) @@ -479,7 +608,7 @@ def merge_info_jsons(keyboard, info_data): # Deep merge certain keys # FIXME(skullydazed/anyone): this should be generalized more so that we can inteligently merge more than one level deep. It would be nice if we could filter on valid keys too. That may have to wait for a future where we use openapi or something. - for key in ('features', 'layout_aliases', 'matrix_pins', 'rgblight', 'usb'): + for key in ('features', 'layout_aliases', 'led_matrix', 'matrix_pins', 'rgblight', 'usb'): if key in new_info_data: if key not in info_data: info_data[key] = {} -- cgit v1.2.3 From b2c26f7cdd4b268e80f98cae7f444956559436ec Mon Sep 17 00:00:00 2001 From: Zach White Date: Tue, 1 Dec 2020 16:04:22 -0800 Subject: get qmk generate-api into a good state --- lib/python/qmk/info.py | 82 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 57 insertions(+), 25 deletions(-) (limited to 'lib/python/qmk/info.py') diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py index 1cf12190d6..39af88f790 100644 --- a/lib/python/qmk/info.py +++ b/lib/python/qmk/info.py @@ -26,13 +26,12 @@ led_matrix_properties = { } rgblight_properties = { - 'led_count': 'RGBLED_NUM', - 'pin': 'RGB_DI_PIN', - 'split_count': 'RGBLED_SPLIT', - 'max_brightness': 'RGBLIGHT_LIMIT_VAL', - 'hue_steps': 'RGBLIGHT_HUE_STEP', - 'saturation_steps': 'RGBLIGHT_SAT_STEP', - 'brightness_steps': 'RGBLIGHT_VAL_STEP' + 'led_count': ('RGBLED_NUM', int), + 'pin': ('RGB_DI_PIN', str), + 'max_brightness': ('RGBLIGHT_LIMIT_VAL', int), + 'hue_steps': ('RGBLIGHT_HUE_STEP', int), + 'saturation_steps': ('RGBLIGHT_SAT_STEP', int), + 'brightness_steps': ('RGBLIGHT_VAL_STEP', int) } rgblight_toggles = { @@ -54,6 +53,8 @@ rgblight_animations = { 'twinkle': 'RGBLIGHT_EFFECT_TWINKLE' } +usb_properties = {'vid': 'VENDOR_ID', 'pid': 'PRODUCT_ID', 'device_ver': 'DEVICE_VER'} + true_values = ['1', 'on', 'yes'] false_values = ['0', 'off', 'no'] @@ -97,7 +98,8 @@ def info_json(keyboard): keyboard_api_validate(info_data) except jsonschema.ValidationError as e: - cli.log.error('Invalid info.json data: %s', e.message) + json_path = '.'.join([str(p) for p in e.absolute_path]) + cli.log.error('Invalid API data: %s: %s: %s', keyboard, json_path, e.message) print(dir(e)) exit() @@ -198,6 +200,9 @@ def _extract_indicators(info_data, config_c): if json_key in info_data.get('indicators', []) and config_key in config_c: _log_warning(info_data, f'Indicator {json_key} is specified in both info.json and config.h, the config.h value wins.') + if 'indicators' not in info_data: + info_data['indicators'] = {} + if config_key in config_c: if 'indicators' not in info_data: info_data['indicators'] = {} @@ -226,10 +231,23 @@ def _extract_community_layouts(info_data, rules): def _extract_features(info_data, rules): """Find all the features enabled in rules.mk. """ + # Special handling for bootmagic which also supports a "lite" mode. + if rules.get('BOOTMAGIC_ENABLE') == 'lite': + rules['BOOTMAGIC_LITE_ENABLE'] = 'on' + del(rules['BOOTMAGIC_ENABLE']) + if rules.get('BOOTMAGIC_ENABLE') == 'full': + rules['BOOTMAGIC_ENABLE'] = 'on' + + # Skip non-boolean features we haven't implemented special handling for + for feature in 'HAPTIC_ENABLE', 'QWIIC_ENABLE': + if rules.get(feature): + del(rules[feature]) + + # Process the rest of the rules as booleans for key, value in rules.items(): if key.endswith('_ENABLE'): key = '_'.join(key.split('_')[:-1]).lower() - value = True if value in true_values else False if value in false_values else value + value = True if value.lower() in true_values else False if value.lower() in false_values else value if 'config_h_features' not in info_data: info_data['config_h_features'] = {} @@ -280,12 +298,21 @@ def _extract_rgblight(info_data, config_c): rgblight = info_data.get('rgblight', {}) animations = rgblight.get('animations', {}) - for json_key, config_key in rgblight_properties.items(): + if 'RGBLED_SPLIT' in config_c: + raw_split = config_c.get('RGBLED_SPLIT', '').replace('{', '').replace('}', '').strip() + rgblight['split_count'] = [int(i) for i in raw_split.split(',')] + + for json_key, config_key_type in rgblight_properties.items(): + config_key, config_type = config_key_type + if config_key in config_c: if json_key in rgblight: _log_warning(info_data, 'RGB Light: %s is specified in both info.json and config.h, the config.h value wins.' % (json_key,)) - rgblight[json_key] = config_c[config_key] + try: + rgblight[json_key] = config_type(config_c[config_key]) + except ValueError as e: + cli.log.error('%s: config.h: Could not convert "%s" to %s: %s', info_data['keyboard_folder'], config_c[config_key], config_type.__name__, e) for json_key, config_key in rgblight_toggles.items(): if config_key in config_c: @@ -332,11 +359,16 @@ def _extract_matrix_info(info_data, config_c): info_data['matrix_pins'] = {} + # FIXME(skullydazed/anyone): Should really check every pin, not just the first if row_pins: - info_data['matrix_pins']['rows'] = row_pins.split(',') + row_pins = [pin.strip() for pin in row_pins.split(',') if pin] + if row_pins[0][0] in 'ABCDEFGHIJK' and row_pins[0][1].isdigit(): + info_data['matrix_pins']['rows'] = row_pins if col_pins: - info_data['matrix_pins']['cols'] = col_pins.split(',') + col_pins = [pin.strip() for pin in col_pins.split(',') if pin] + if col_pins[0][0] in 'ABCDEFGHIJK' and col_pins[0][1].isdigit(): + info_data['matrix_pins']['cols'] = col_pins if direct_pins: if 'matrix_pins' in info_data: @@ -345,6 +377,9 @@ def _extract_matrix_info(info_data, config_c): info_data['matrix_pins'] = {} direct_pin_array = [] + while direct_pins[-1] != '}': + direct_pins = direct_pins[:-1] + for row in direct_pins.split('},{'): if row.startswith('{'): row = row[1:] @@ -368,8 +403,6 @@ def _extract_matrix_info(info_data, config_c): def _extract_usb_info(info_data, config_c): """Populate the USB information. """ - usb_properties = {'vid': 'VENDOR_ID', 'pid': 'PRODUCT_ID', 'device_ver': 'DEVICE_VER'} - if 'usb' not in info_data: info_data['usb'] = {} @@ -378,10 +411,7 @@ def _extract_usb_info(info_data, config_c): if info_name in info_data['usb']: _log_warning(info_data, '%s in config.h is overwriting usb.%s in info.json' % (config_name, info_name)) - info_data['usb'][info_name] = config_c[config_name] - - elif info_name not in info_data['usb']: - _log_error(info_data, '%s not specified in config.h, and %s not specified in info.json. One is required.' % (config_name, info_name)) + info_data['usb'][info_name] = '0x' + config_c[config_name][2:].upper() return info_data @@ -519,8 +549,9 @@ def arm_processor_rules(info_data, rules): info_data['processor'] = 'unknown' if 'BOOTLOADER' in rules: - if 'bootloader' in info_data: - _log_warning(info_data, 'Bootloader is specified in both info.json and rules.mk, the rules.mk value wins.') + # FIXME(skullydazed/anyone): need to remove the massive amounts of duplication first + # if 'bootloader' in info_data: + # _log_warning(info_data, 'Bootloader is specified in both info.json and rules.mk, the rules.mk value wins.') info_data['bootloader'] = rules['BOOTLOADER'] @@ -558,8 +589,9 @@ def avr_processor_rules(info_data, rules): info_data['processor'] = 'unknown' if 'BOOTLOADER' in rules: - if 'bootloader' in info_data: - _log_warning(info_data, 'Bootloader is specified in both info.json and rules.mk, the rules.mk value wins.') + # FIXME(skullydazed/anyone): need to remove the massive amounts of duplication first + # if 'bootloader' in info_data: + # _log_warning(info_data, 'Bootloader is specified in both info.json and rules.mk, the rules.mk value wins.') info_data['bootloader'] = rules['BOOTLOADER'] else: @@ -593,8 +625,8 @@ def merge_info_jsons(keyboard, info_data): keyboard_validate(new_info_data) except jsonschema.ValidationError as e: - cli.log.error('Invalid info.json data: %s', e.message) - cli.log.error('Not including file %s', info_file) + json_path = '.'.join([str(p) for p in e.absolute_path]) + cli.log.error('Invalid info.json data: %s: %s: %s', info_file, json_path, e.message) continue if not isinstance(new_info_data, dict): -- cgit v1.2.3 From 56ef80216ae4c67e2a70857c61d1e62eec1ab380 Mon Sep 17 00:00:00 2001 From: Zach White Date: Wed, 30 Dec 2020 11:21:18 -0800 Subject: make flake8 happy --- lib/python/qmk/info.py | 84 ++++++++++++++++++++++++++++---------------------- 1 file changed, 47 insertions(+), 37 deletions(-) (limited to 'lib/python/qmk/info.py') diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py index 39af88f790..efd339115b 100644 --- a/lib/python/qmk/info.py +++ b/lib/python/qmk/info.py @@ -234,14 +234,14 @@ def _extract_features(info_data, rules): # Special handling for bootmagic which also supports a "lite" mode. if rules.get('BOOTMAGIC_ENABLE') == 'lite': rules['BOOTMAGIC_LITE_ENABLE'] = 'on' - del(rules['BOOTMAGIC_ENABLE']) + del rules['BOOTMAGIC_ENABLE'] if rules.get('BOOTMAGIC_ENABLE') == 'full': rules['BOOTMAGIC_ENABLE'] = 'on' # Skip non-boolean features we haven't implemented special handling for for feature in 'HAPTIC_ENABLE', 'QWIIC_ENABLE': if rules.get(feature): - del(rules[feature]) + del rules[feature] # Process the rest of the rules as booleans for key, value in rules.items(): @@ -337,6 +337,45 @@ def _extract_rgblight(info_data, config_c): return info_data +def _extract_pins(pins): + """Returns a list of pins from a comma separated string of pins. + """ + pins = [pin.strip() for pin in pins.split(',') if pin] + + for pin in pins: + if pin[0] not in 'ABCDEFGHIJK' or not pin[1].isdigit(): + raise ValueError(f'Invalid pin: {pin}') + + return pins + + +def _extract_direct_matrix(info_data, direct_pins): + """ + """ + info_data['matrix_pins'] = {} + direct_pin_array = [] + + while direct_pins[-1] != '}': + direct_pins = direct_pins[:-1] + + for row in direct_pins.split('},{'): + if row.startswith('{'): + row = row[1:] + + if row.endswith('}'): + row = row[:-1] + + direct_pin_array.append([]) + + for pin in row.split(','): + if pin == 'NO_PIN': + pin = None + + direct_pin_array[-1].append(pin) + + return direct_pin_array + + def _extract_matrix_info(info_data, config_c): """Populate the matrix information. """ @@ -349,53 +388,24 @@ def _extract_matrix_info(info_data, config_c): _log_warning(info_data, 'Matrix size is specified in both info.json and config.h, the config.h values win.') info_data['matrix_size'] = { - 'rows': compute(config_c.get('MATRIX_ROWS', '0')), 'cols': compute(config_c.get('MATRIX_COLS', '0')), + 'rows': compute(config_c.get('MATRIX_ROWS', '0')), } if row_pins and col_pins: if 'matrix_pins' in info_data: _log_warning(info_data, 'Matrix pins are specified in both info.json and config.h, the config.h values win.') - info_data['matrix_pins'] = {} - - # FIXME(skullydazed/anyone): Should really check every pin, not just the first - if row_pins: - row_pins = [pin.strip() for pin in row_pins.split(',') if pin] - if row_pins[0][0] in 'ABCDEFGHIJK' and row_pins[0][1].isdigit(): - info_data['matrix_pins']['rows'] = row_pins - - if col_pins: - col_pins = [pin.strip() for pin in col_pins.split(',') if pin] - if col_pins[0][0] in 'ABCDEFGHIJK' and col_pins[0][1].isdigit(): - info_data['matrix_pins']['cols'] = col_pins + info_data['matrix_pins'] = { + 'cols': _extract_pins(col_pins), + 'rows': _extract_pins(row_pins), + } if direct_pins: if 'matrix_pins' in info_data: _log_warning(info_data, 'Direct pins are specified in both info.json and config.h, the config.h values win.') - info_data['matrix_pins'] = {} - direct_pin_array = [] - - while direct_pins[-1] != '}': - direct_pins = direct_pins[:-1] - - for row in direct_pins.split('},{'): - if row.startswith('{'): - row = row[1:] - - if row.endswith('}'): - row = row[:-1] - - direct_pin_array.append([]) - - for pin in row.split(','): - if pin == 'NO_PIN': - pin = None - - direct_pin_array[-1].append(pin) - - info_data['matrix_pins']['direct'] = direct_pin_array + info_data['matrix_pins']['direct'] = _extract_direct_matrix(info_data, direct_pins) return info_data -- cgit v1.2.3 From b44532aa3dbdce702d662e5b1cf043e461846d8b Mon Sep 17 00:00:00 2001 From: Zach White Date: Wed, 6 Jan 2021 10:28:58 -0800 Subject: change invalid pin from error to warning --- lib/python/qmk/info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/python/qmk/info.py') diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py index efd339115b..89c08809ba 100644 --- a/lib/python/qmk/info.py +++ b/lib/python/qmk/info.py @@ -344,7 +344,7 @@ def _extract_pins(pins): for pin in pins: if pin[0] not in 'ABCDEFGHIJK' or not pin[1].isdigit(): - raise ValueError(f'Invalid pin: {pin}') + cli.log.warning(f'Nonstandard pin format: {pin}') return pins -- cgit v1.2.3 From e2e793c1c3efeb96108072c2c8a57555c0fa76a0 Mon Sep 17 00:00:00 2001 From: Zach White Date: Wed, 6 Jan 2021 10:37:31 -0800 Subject: Revert "change invalid pin from error to warning" This reverts commit 68da9cd19658071f0c2f22e97956303281703176. --- lib/python/qmk/info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/python/qmk/info.py') diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py index 89c08809ba..efd339115b 100644 --- a/lib/python/qmk/info.py +++ b/lib/python/qmk/info.py @@ -344,7 +344,7 @@ def _extract_pins(pins): for pin in pins: if pin[0] not in 'ABCDEFGHIJK' or not pin[1].isdigit(): - cli.log.warning(f'Nonstandard pin format: {pin}') + raise ValueError(f'Invalid pin: {pin}') return pins -- cgit v1.2.3 From 30331b383f9ef4620e47aa07e4f9af7fae9d30b3 Mon Sep 17 00:00:00 2001 From: Zach White Date: Fri, 8 Jan 2021 00:00:15 -0800 Subject: fix bugs triggered by certain boards --- lib/python/qmk/info.py | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) (limited to 'lib/python/qmk/info.py') diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py index efd339115b..28c281a4bc 100644 --- a/lib/python/qmk/info.py +++ b/lib/python/qmk/info.py @@ -315,11 +315,10 @@ def _extract_rgblight(info_data, config_c): cli.log.error('%s: config.h: Could not convert "%s" to %s: %s', info_data['keyboard_folder'], config_c[config_key], config_type.__name__, e) for json_key, config_key in rgblight_toggles.items(): - if config_key in config_c: - if json_key in rgblight: - _log_warning(info_data, 'RGB Light: %s is specified in both info.json and config.h, the config.h value wins.', json_key) + if config_key in config_c and json_key in rgblight: + _log_warning(info_data, 'RGB Light: %s is specified in both info.json and config.h, the config.h value wins.', json_key) - rgblight[json_key] = config_c[config_key] + rgblight[json_key] = config_key in config_c for json_key, config_key in rgblight_animations.items(): if config_key in config_c: @@ -337,16 +336,30 @@ def _extract_rgblight(info_data, config_c): return info_data -def _extract_pins(pins): - """Returns a list of pins from a comma separated string of pins. +def _pin_name(pin): + """Returns the proper representation for a pin. """ - pins = [pin.strip() for pin in pins.split(',') if pin] + pin = pin.strip() + + if not pin: + return None + + elif pin.isdigit(): + return int(pin) - for pin in pins: - if pin[0] not in 'ABCDEFGHIJK' or not pin[1].isdigit(): - raise ValueError(f'Invalid pin: {pin}') + elif pin == 'NO_PIN': + return None - return pins + elif pin[0] in 'ABCDEFGHIJK' and pin[1].isdigit(): + return pin + + raise ValueError(f'Invalid pin: {pin}') + + +def _extract_pins(pins): + """Returns a list of pins from a comma separated string of pins. + """ + return [_pin_name(pin) for pin in pins.split(',')] def _extract_direct_matrix(info_data, direct_pins): -- cgit v1.2.3 From 962bc8d9dd413690dbeadeaac971a5389697210f Mon Sep 17 00:00:00 2001 From: Zach White Date: Sat, 9 Jan 2021 13:34:14 -0800 Subject: Use the schema to eliminate custom code (#11108) * use the schema to eliminate custom code * Update docs/reference_info_json.md Co-authored-by: Ryan * make flake8 happy * bugfix * do not overwrite make vars from json Co-authored-by: Ryan --- lib/python/qmk/info.py | 60 ++++++++++++++++++++++++-------------------------- 1 file changed, 29 insertions(+), 31 deletions(-) (limited to 'lib/python/qmk/info.py') diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py index 28c281a4bc..0ea5136a97 100644 --- a/lib/python/qmk/info.py +++ b/lib/python/qmk/info.py @@ -1,6 +1,7 @@ """Functions that help us generate and use info.json files. """ import json +from collections.abc import Mapping from glob import glob from pathlib import Path @@ -140,6 +141,8 @@ def _json_load(json_file): def _jsonschema(schema_name): """Read a jsonschema file from disk. + + FIXME(skullydazed/anyone): Refactor to make this a public function. """ schema_path = Path(f'data/schemas/{schema_name}.jsonschema') @@ -638,49 +641,44 @@ def unknown_processor_rules(info_data, rules): return info_data +def deep_update(origdict, newdict): + """Update a dictionary in place, recursing to do a deep copy. + """ + for key, value in newdict.items(): + if isinstance(value, Mapping): + origdict[key] = deep_update(origdict.get(key, {}), value) + + else: + origdict[key] = value + + return origdict + + def merge_info_jsons(keyboard, info_data): """Return a merged copy of all the info.json files for a keyboard. """ for info_file in find_info_json(keyboard): # Load and validate the JSON data + new_info_data = _json_load(info_file) + + if not isinstance(new_info_data, dict): + _log_error(info_data, "Invalid file %s, root object should be a dictionary." % (str(info_file),)) + continue + try: - new_info_data = _json_load(info_file) keyboard_validate(new_info_data) - except jsonschema.ValidationError as e: json_path = '.'.join([str(p) for p in e.absolute_path]) - cli.log.error('Invalid info.json data: %s: %s: %s', info_file, json_path, e.message) + cli.log.error('Not including data from file: %s', info_file) + cli.log.error('\t%s: %s', json_path, e.message) continue - if not isinstance(new_info_data, dict): - _log_error(info_data, "Invalid file %s, root object should be a dictionary." % (str(info_file),)) - continue - - # Copy whitelisted keys into `info_data` - for key in ('debounce', 'diode_direction', 'indicators', 'keyboard_name', 'manufacturer', 'identifier', 'url', 'maintainer', 'processor', 'bootloader', 'width', 'height'): - if key in new_info_data: - info_data[key] = new_info_data[key] - - # Deep merge certain keys - # FIXME(skullydazed/anyone): this should be generalized more so that we can inteligently merge more than one level deep. It would be nice if we could filter on valid keys too. That may have to wait for a future where we use openapi or something. - for key in ('features', 'layout_aliases', 'led_matrix', 'matrix_pins', 'rgblight', 'usb'): - if key in new_info_data: - if key not in info_data: - info_data[key] = {} - - info_data[key].update(new_info_data[key]) - - # Merge the layouts - if 'community_layouts' in new_info_data: - if 'community_layouts' in info_data: - for layout in new_info_data['community_layouts']: - if layout not in info_data['community_layouts']: - info_data['community_layouts'].append(layout) - else: - info_data['community_layouts'] = new_info_data['community_layouts'] + # Mark the layouts as coming from json + for layout in new_info_data.get('layouts', {}).values(): + layout['c_macro'] = False - if 'layouts' in new_info_data: - _merge_layouts(info_data, new_info_data) + # Update info_data with the new data + deep_update(info_data, new_info_data) return info_data -- cgit v1.2.3 From eaa9106ec74591593e638ac015a5c90d17b30612 Mon Sep 17 00:00:00 2001 From: Zach White Date: Sat, 9 Jan 2021 20:18:47 -0800 Subject: Add support for specifying BOARD in info.json (#11492) * add support for specifying BOARD in info.json * move BOARD from rules.mk to info.json for clueboard * fix keyboards that do not require board * remove out of compliance values --- lib/python/qmk/info.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'lib/python/qmk/info.py') diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py index 0ea5136a97..cc81f7a086 100644 --- a/lib/python/qmk/info.py +++ b/lib/python/qmk/info.py @@ -462,7 +462,7 @@ def _extract_rules_mk(info_data): """Pull some keyboard information from existing rules.mk files """ rules = rules_mk(info_data['keyboard_folder']) - mcu = rules.get('MCU') + mcu = rules.get('MCU', info_data.get('processor')) if mcu in CHIBIOS_PROCESSORS: arm_processor_rules(info_data, rules) @@ -594,6 +594,12 @@ def arm_processor_rules(info_data, rules): elif 'ARM_ATSAM' in rules: info_data['platform'] = 'ARM_ATSAM' + if 'BOARD' in rules: + if 'board' in info_data: + _log_warning(info_data, 'Board is specified in both info.json and rules.mk, the rules.mk value wins.') + + info_data['board'] = rules['BOARD'] + return info_data -- cgit v1.2.3 From ef6329af7c7be77b537fbfc5a5cc7105acc679f7 Mon Sep 17 00:00:00 2001 From: Zach White Date: Sun, 31 Jan 2021 12:46:00 -0800 Subject: Create a system to map between info.json and config.h/rules.mk (#11548) * generate rules.mk from a json mapping * generate rules.mk from a json mapping * support for config.h from json maps * improve the mapping system * document the mapping system * move data/maps to data/mappings * fix flake8 errors * fixup LED_MATRIX_DRIVER * remove product and description from the vision_division keymap level * reduce the complexity of generate-rules-mk * add tests for the generate commands * fix qmk doctor when submodules are not clean --- lib/python/qmk/info.py | 343 +++++++++++++++---------------------------------- 1 file changed, 102 insertions(+), 241 deletions(-) (limited to 'lib/python/qmk/info.py') diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py index cc81f7a086..2accaba9e4 100644 --- a/lib/python/qmk/info.py +++ b/lib/python/qmk/info.py @@ -5,57 +5,18 @@ from collections.abc import Mapping from glob import glob from pathlib import Path +import hjson import jsonschema +from dotty_dict import dotty from milc import cli -from qmk.constants import CHIBIOS_PROCESSORS, LUFA_PROCESSORS, VUSB_PROCESSORS, LED_INDICATORS +from qmk.constants import CHIBIOS_PROCESSORS, LUFA_PROCESSORS, VUSB_PROCESSORS from qmk.c_parse import find_layouts from qmk.keyboard import config_h, rules_mk from qmk.keymap import list_keymaps from qmk.makefile import parse_rules_mk_file from qmk.math import compute -led_matrix_properties = { - 'driver_count': 'LED_DRIVER_COUNT', - 'driver_addr1': 'LED_DRIVER_ADDR_1', - 'driver_addr2': 'LED_DRIVER_ADDR_2', - 'driver_addr3': 'LED_DRIVER_ADDR_3', - 'driver_addr4': 'LED_DRIVER_ADDR_4', - 'led_count': 'LED_DRIVER_LED_COUNT', - 'timeout': 'ISSI_TIMEOUT', - 'persistence': 'ISSI_PERSISTENCE' -} - -rgblight_properties = { - 'led_count': ('RGBLED_NUM', int), - 'pin': ('RGB_DI_PIN', str), - 'max_brightness': ('RGBLIGHT_LIMIT_VAL', int), - 'hue_steps': ('RGBLIGHT_HUE_STEP', int), - 'saturation_steps': ('RGBLIGHT_SAT_STEP', int), - 'brightness_steps': ('RGBLIGHT_VAL_STEP', int) -} - -rgblight_toggles = { - 'sleep': 'RGBLIGHT_SLEEP', - 'split': 'RGBLIGHT_SPLIT', -} - -rgblight_animations = { - 'all': 'RGBLIGHT_ANIMATIONS', - 'alternating': 'RGBLIGHT_EFFECT_ALTERNATING', - 'breathing': 'RGBLIGHT_EFFECT_BREATHING', - 'christmas': 'RGBLIGHT_EFFECT_CHRISTMAS', - 'knight': 'RGBLIGHT_EFFECT_KNIGHT', - 'rainbow_mood': 'RGBLIGHT_EFFECT_RAINBOW_MOOD', - 'rainbow_swirl': 'RGBLIGHT_EFFECT_RAINBOW_SWIRL', - 'rgb_test': 'RGBLIGHT_EFFECT_RGB_TEST', - 'snake': 'RGBLIGHT_EFFECT_SNAKE', - 'static_gradient': 'RGBLIGHT_EFFECT_STATIC_GRADIENT', - 'twinkle': 'RGBLIGHT_EFFECT_TWINKLE' -} - -usb_properties = {'vid': 'VENDOR_ID', 'pid': 'PRODUCT_ID', 'device_ver': 'DEVICE_VER'} - true_values = ['1', 'on', 'yes'] false_values = ['0', 'off', 'no'] @@ -101,7 +62,6 @@ def info_json(keyboard): except jsonschema.ValidationError as e: json_path = '.'.join([str(p) for p in e.absolute_path]) cli.log.error('Invalid API data: %s: %s: %s', keyboard, json_path, e.message) - print(dir(e)) exit() # Make sure we have at least one layout @@ -132,7 +92,7 @@ def _json_load(json_file): Note: file must be a Path object. """ try: - return json.load(json_file.open()) + return hjson.load(json_file.open()) except json.decoder.JSONDecodeError as e: cli.log.error('Invalid JSON encountered attempting to load {fg_cyan}%s{fg_reset}:\n\t{fg_red}%s', json_file, e) @@ -172,65 +132,6 @@ def keyboard_api_validate(data): return validator(data) -def _extract_debounce(info_data, config_c): - """Handle debounce. - """ - if 'debounce' in info_data and 'DEBOUNCE' in config_c: - _log_warning(info_data, 'Debounce is specified in both info.json and config.h, the config.h value wins.') - - if 'DEBOUNCE' in config_c: - info_data['debounce'] = int(config_c['DEBOUNCE']) - - return info_data - - -def _extract_diode_direction(info_data, config_c): - """Handle the diode direction. - """ - if 'diode_direction' in info_data and 'DIODE_DIRECTION' in config_c: - _log_warning(info_data, 'Diode direction is specified in both info.json and config.h, the config.h value wins.') - - if 'DIODE_DIRECTION' in config_c: - info_data['diode_direction'] = config_c.get('DIODE_DIRECTION') - - return info_data - - -def _extract_indicators(info_data, config_c): - """Find the LED indicator information. - """ - for json_key, config_key in LED_INDICATORS.items(): - if json_key in info_data.get('indicators', []) and config_key in config_c: - _log_warning(info_data, f'Indicator {json_key} is specified in both info.json and config.h, the config.h value wins.') - - if 'indicators' not in info_data: - info_data['indicators'] = {} - - if config_key in config_c: - if 'indicators' not in info_data: - info_data['indicators'] = {} - - info_data['indicators'][json_key] = config_c.get(config_key) - - return info_data - - -def _extract_community_layouts(info_data, rules): - """Find the community layouts in rules.mk. - """ - community_layouts = rules['LAYOUTS'].split() if 'LAYOUTS' in rules else [] - - if 'community_layouts' in info_data: - for layout in community_layouts: - if layout not in info_data['community_layouts']: - community_layouts.append(layout) - - else: - info_data['community_layouts'] = community_layouts - - return info_data - - def _extract_features(info_data, rules): """Find all the features enabled in rules.mk. """ @@ -267,78 +168,6 @@ def _extract_features(info_data, rules): return info_data -def _extract_led_drivers(info_data, rules): - """Find all the LED drivers set in rules.mk. - """ - if 'LED_MATRIX_DRIVER' in rules: - if 'led_matrix' not in info_data: - info_data['led_matrix'] = {} - - if info_data['led_matrix'].get('driver'): - _log_warning(info_data, 'LED Matrix driver is specified in both info.json and rules.mk, the rules.mk value wins.') - - info_data['led_matrix']['driver'] = rules['LED_MATRIX_DRIVER'] - - return info_data - - -def _extract_led_matrix(info_data, config_c): - """Handle the led_matrix configuration. - """ - led_matrix = info_data.get('led_matrix', {}) - - for json_key, config_key in led_matrix_properties.items(): - if config_key in config_c: - if json_key in led_matrix: - _log_warning(info_data, 'LED Matrix: %s is specified in both info.json and config.h, the config.h value wins.' % (json_key,)) - - led_matrix[json_key] = config_c[config_key] - - -def _extract_rgblight(info_data, config_c): - """Handle the rgblight configuration. - """ - rgblight = info_data.get('rgblight', {}) - animations = rgblight.get('animations', {}) - - if 'RGBLED_SPLIT' in config_c: - raw_split = config_c.get('RGBLED_SPLIT', '').replace('{', '').replace('}', '').strip() - rgblight['split_count'] = [int(i) for i in raw_split.split(',')] - - for json_key, config_key_type in rgblight_properties.items(): - config_key, config_type = config_key_type - - if config_key in config_c: - if json_key in rgblight: - _log_warning(info_data, 'RGB Light: %s is specified in both info.json and config.h, the config.h value wins.' % (json_key,)) - - try: - rgblight[json_key] = config_type(config_c[config_key]) - except ValueError as e: - cli.log.error('%s: config.h: Could not convert "%s" to %s: %s', info_data['keyboard_folder'], config_c[config_key], config_type.__name__, e) - - for json_key, config_key in rgblight_toggles.items(): - if config_key in config_c and json_key in rgblight: - _log_warning(info_data, 'RGB Light: %s is specified in both info.json and config.h, the config.h value wins.', json_key) - - rgblight[json_key] = config_key in config_c - - for json_key, config_key in rgblight_animations.items(): - if config_key in config_c: - if json_key in animations: - _log_warning(info_data, 'RGB Light: animations: %s is specified in both info.json and config.h, the config.h value wins.' % (json_key,)) - - animations[json_key] = config_c[config_key] - - if animations: - rgblight['animations'] = animations - - if rgblight: - info_data['rgblight'] = rgblight - - return info_data - - def _pin_name(pin): """Returns the proper representation for a pin. """ @@ -426,34 +255,59 @@ def _extract_matrix_info(info_data, config_c): return info_data -def _extract_usb_info(info_data, config_c): - """Populate the USB information. +def _extract_config_h(info_data): + """Pull some keyboard information from existing config.h files """ - if 'usb' not in info_data: - info_data['usb'] = {} + config_c = config_h(info_data['keyboard_folder']) - for info_name, config_name in usb_properties.items(): - if config_name in config_c: - if info_name in info_data['usb']: - _log_warning(info_data, '%s in config.h is overwriting usb.%s in info.json' % (config_name, info_name)) + # Pull in data from the json map + dotty_info = dotty(info_data) + info_config_map = _json_load(Path('data/mappings/info_config.json')) - info_data['usb'][info_name] = '0x' + config_c[config_name][2:].upper() + for config_key, info_dict in info_config_map.items(): + info_key = info_dict['info_key'] + key_type = info_dict.get('value_type', 'str') - return info_data + try: + if config_key in config_c and info_dict.get('to_json', True): + if dotty_info.get(info_key) and info_dict.get('warn_duplicate', True): + _log_warning(info_data, '%s in config.h is overwriting %s in info.json' % (config_key, info_key)) + if key_type.startswith('array'): + if '.' in key_type: + key_type, array_type = key_type.split('.', 1) + else: + array_type = None -def _extract_config_h(info_data): - """Pull some keyboard information from existing config.h files - """ - config_c = config_h(info_data['keyboard_folder']) + config_value = config_c[config_key].replace('{', '').replace('}', '').strip() + + if array_type == 'int': + dotty_info[info_key] = list(map(int, config_value.split(','))) + else: + dotty_info[info_key] = config_value.split(',') + + elif key_type == 'bool': + dotty_info[info_key] = config_c[config_key] in true_values + + elif key_type == 'hex': + dotty_info[info_key] = '0x' + config_c[config_key][2:].upper() - _extract_debounce(info_data, config_c) - _extract_diode_direction(info_data, config_c) - _extract_indicators(info_data, config_c) + elif key_type == 'list': + dotty_info[info_key] = config_c[config_key].split() + + elif key_type == 'int': + dotty_info[info_key] = int(config_c[config_key]) + + else: + dotty_info[info_key] = config_c[config_key] + + except Exception as e: + _log_warning(info_data, f'{config_key}->{info_key}: {e}') + + info_data.update(dotty_info) + + # Pull data that easily can't be mapped in json _extract_matrix_info(info_data, config_c) - _extract_usb_info(info_data, config_c) - _extract_led_matrix(info_data, config_c) - _extract_rgblight(info_data, config_c) return info_data @@ -462,21 +316,66 @@ def _extract_rules_mk(info_data): """Pull some keyboard information from existing rules.mk files """ rules = rules_mk(info_data['keyboard_folder']) - mcu = rules.get('MCU', info_data.get('processor')) + info_data['processor'] = rules.get('MCU', info_data.get('processor', 'atmega32u4')) - if mcu in CHIBIOS_PROCESSORS: + if info_data['processor'] in CHIBIOS_PROCESSORS: arm_processor_rules(info_data, rules) - elif mcu in LUFA_PROCESSORS + VUSB_PROCESSORS: + elif info_data['processor'] in LUFA_PROCESSORS + VUSB_PROCESSORS: avr_processor_rules(info_data, rules) else: - cli.log.warning("%s: Unknown MCU: %s" % (info_data['keyboard_folder'], mcu)) + cli.log.warning("%s: Unknown MCU: %s" % (info_data['keyboard_folder'], info_data['processor'])) unknown_processor_rules(info_data, rules) - _extract_community_layouts(info_data, rules) + # Pull in data from the json map + dotty_info = dotty(info_data) + info_rules_map = _json_load(Path('data/mappings/info_rules.json')) + + for rules_key, info_dict in info_rules_map.items(): + info_key = info_dict['info_key'] + key_type = info_dict.get('value_type', 'str') + + try: + if rules_key in rules and info_dict.get('to_json', True): + if dotty_info.get(info_key) and info_dict.get('warn_duplicate', True): + _log_warning(info_data, '%s in rules.mk is overwriting %s in info.json' % (rules_key, info_key)) + + if key_type.startswith('array'): + if '.' in key_type: + key_type, array_type = key_type.split('.', 1) + else: + array_type = None + + rules_value = rules[rules_key].replace('{', '').replace('}', '').strip() + + if array_type == 'int': + dotty_info[info_key] = list(map(int, rules_value.split(','))) + else: + dotty_info[info_key] = rules_value.split(',') + + elif key_type == 'list': + dotty_info[info_key] = rules[rules_key].split() + + elif key_type == 'bool': + dotty_info[info_key] = rules[rules_key] in true_values + + elif key_type == 'hex': + dotty_info[info_key] = '0x' + rules[rules_key][2:].upper() + + elif key_type == 'int': + dotty_info[info_key] = int(rules[rules_key]) + + else: + dotty_info[info_key] = rules[rules_key] + + except Exception as e: + _log_warning(info_data, f'{rules_key}->{info_key}: {e}') + + info_data.update(dotty_info) + + # Merge in config values that can't be easily mapped _extract_features(info_data, rules) - _extract_led_drivers(info_data, rules) return info_data @@ -565,23 +464,7 @@ def arm_processor_rules(info_data, rules): info_data['processor_type'] = 'arm' info_data['protocol'] = 'ChibiOS' - if 'MCU' in rules: - if 'processor' in info_data: - _log_warning(info_data, 'Processor/MCU is specified in both info.json and rules.mk, the rules.mk value wins.') - - info_data['processor'] = rules['MCU'] - - elif 'processor' not in info_data: - info_data['processor'] = 'unknown' - - if 'BOOTLOADER' in rules: - # FIXME(skullydazed/anyone): need to remove the massive amounts of duplication first - # if 'bootloader' in info_data: - # _log_warning(info_data, 'Bootloader is specified in both info.json and rules.mk, the rules.mk value wins.') - - info_data['bootloader'] = rules['BOOTLOADER'] - - else: + if 'bootloader' not in info_data: if 'STM32' in info_data['processor']: info_data['bootloader'] = 'stm32-dfu' else: @@ -594,12 +477,6 @@ def arm_processor_rules(info_data, rules): elif 'ARM_ATSAM' in rules: info_data['platform'] = 'ARM_ATSAM' - if 'BOARD' in rules: - if 'board' in info_data: - _log_warning(info_data, 'Board is specified in both info.json and rules.mk, the rules.mk value wins.') - - info_data['board'] = rules['BOARD'] - return info_data @@ -607,26 +484,10 @@ def avr_processor_rules(info_data, rules): """Setup the default info for an AVR board. """ info_data['processor_type'] = 'avr' - info_data['bootloader'] = rules['BOOTLOADER'] if 'BOOTLOADER' in rules else 'atmel-dfu' info_data['platform'] = rules['ARCH'] if 'ARCH' in rules else 'unknown' info_data['protocol'] = 'V-USB' if rules.get('MCU') in VUSB_PROCESSORS else 'LUFA' - if 'MCU' in rules: - if 'processor' in info_data: - _log_warning(info_data, 'Processor/MCU is specified in both info.json and rules.mk, the rules.mk value wins.') - - info_data['processor'] = rules['MCU'] - - elif 'processor' not in info_data: - info_data['processor'] = 'unknown' - - if 'BOOTLOADER' in rules: - # FIXME(skullydazed/anyone): need to remove the massive amounts of duplication first - # if 'bootloader' in info_data: - # _log_warning(info_data, 'Bootloader is specified in both info.json and rules.mk, the rules.mk value wins.') - - info_data['bootloader'] = rules['BOOTLOADER'] - else: + if 'bootloader' not in info_data: info_data['bootloader'] = 'atmel-dfu' # FIXME(fauxpark/anyone): Eventually we should detect the protocol by looking at PROTOCOL inherited from mcu_selection.mk: -- cgit v1.2.3 From 1581ea48dcd48d0d3f42cc09b388c468aedec45d Mon Sep 17 00:00:00 2001 From: Zach White Date: Sat, 27 Feb 2021 12:00:50 -0800 Subject: Fix develop (#12039) Fixes file encoding errors on Windows, and layouts not correctly merging into info.json. * force utf8 encoding * correctly merge layouts and layout aliases * show what aliases point to --- lib/python/qmk/info.py | 44 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 10 deletions(-) (limited to 'lib/python/qmk/info.py') diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py index 2accaba9e4..cf5dc6640b 100644 --- a/lib/python/qmk/info.py +++ b/lib/python/qmk/info.py @@ -45,7 +45,12 @@ def info_json(keyboard): info_data['keymaps'][keymap.name] = {'url': f'https://raw.githubusercontent.com/qmk/qmk_firmware/master/{keymap}/keymap.json'} # Populate layout data - for layout_name, layout_json in _find_all_layouts(info_data, keyboard).items(): + layouts, aliases = _find_all_layouts(info_data, keyboard) + + if aliases: + info_data['layout_aliases'] = aliases + + for layout_name, layout_json in layouts.items(): if not layout_name.startswith('LAYOUT_kc'): layout_json['c_macro'] = True info_data['layouts'][layout_name] = layout_json @@ -92,7 +97,7 @@ def _json_load(json_file): Note: file must be a Path object. """ try: - return hjson.load(json_file.open()) + return hjson.load(json_file.open(encoding='utf-8')) except json.decoder.JSONDecodeError as e: cli.log.error('Invalid JSON encountered attempting to load {fg_cyan}%s{fg_reset}:\n\t{fg_red}%s', json_file, e) @@ -415,21 +420,28 @@ def _merge_layouts(info_data, new_info_data): def _search_keyboard_h(path): current_path = Path('keyboards/') + aliases = {} layouts = {} + for directory in path.parts: current_path = current_path / directory keyboard_h = '%s.h' % (directory,) keyboard_h_path = current_path / keyboard_h if keyboard_h_path.exists(): - layouts.update(find_layouts(keyboard_h_path)) + new_layouts, new_aliases = find_layouts(keyboard_h_path) + layouts.update(new_layouts) + + for alias, alias_text in new_aliases.items(): + if alias_text in layouts: + aliases[alias] = alias_text - return layouts + return layouts, aliases def _find_all_layouts(info_data, keyboard): """Looks for layout macros associated with this keyboard. """ - layouts = _search_keyboard_h(Path(keyboard)) + layouts, aliases = _search_keyboard_h(Path(keyboard)) if not layouts: # If we don't find any layouts from info.json or keyboard.h we widen our search. This is error prone which is why we want to encourage people to follow the standard above. @@ -437,11 +449,15 @@ def _find_all_layouts(info_data, keyboard): for file in glob('keyboards/%s/*.h' % keyboard): if file.endswith('.h'): - these_layouts = find_layouts(file) + these_layouts, these_aliases = find_layouts(file) + if these_layouts: layouts.update(these_layouts) - return layouts + if these_aliases: + aliases.update(these_aliases) + + return layouts, aliases def _log_error(info_data, message): @@ -540,11 +556,19 @@ def merge_info_jsons(keyboard, info_data): cli.log.error('\t%s: %s', json_path, e.message) continue - # Mark the layouts as coming from json - for layout in new_info_data.get('layouts', {}).values(): - layout['c_macro'] = False + # Merge layout data in + for layout_name, layout in new_info_data.get('layouts', {}).items(): + if layout_name in info_data['layouts']: + for new_key, existing_key in zip(layout['layout'], info_data['layouts'][layout_name]['layout']): + existing_key.update(new_key) + else: + layout['c_macro'] = False + info_data['layouts'][layout_name] = layout # Update info_data with the new data + if 'layouts' in new_info_data: + del (new_info_data['layouts']) + deep_update(info_data, new_info_data) return info_data -- cgit v1.2.3