Merge remote-tracking branch 'upstream/master'

31 files changed, 1006 insertions, 0 deletions

diff --git a/lib/python/qmk/__init__.py b/lib/python/qmk/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/lib/python/qmk/__init__.py
diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py
new file mode 100644
index 0000000000..72ee38f562
--- /dev/null
+++ b/lib/python/qmk/cli/__init__.py
@@ -0,0 +1,17 @@
+"""QMK CLI Subcommands
+
+We list each subcommand here explicitly because all the reliable ways of searching for modules are slow and delay startup.
+"""
+from . import cformat
+from . import compile
+from . import config
+from . import docs
+from . import doctor
+from . import flash
+from . import hello
+from . import json
+from . import list
+from . import kle2json
+from . import new
+from . import pyformat
+from . import pytest
diff --git a/lib/python/qmk/cli/cformat.py b/lib/python/qmk/cli/cformat.py
new file mode 100644
index 0000000000..17ca91b3b5
--- /dev/null
+++ b/lib/python/qmk/cli/cformat.py
@@ -0,0 +1,43 @@
+"""Format C code according to QMK's style.
+"""
+import os
+import subprocess
+from shutil import which
+
+from milc import cli
+
+
+@cli.argument('files', nargs='*', arg_only=True, help='Filename(s) to format.')
+@cli.subcommand("Format C code according to QMK's style.")
+def cformat(cli):
+    """Format C code according to QMK's style.
+    """
+    # Determine which version of clang-format to use
+    clang_format = ['clang-format', '-i']
+    for clang_version in [10, 9, 8, 7]:
+        binary = 'clang-format-%d' % clang_version
+        if which(binary):
+            clang_format[0] = binary
+            break
+
+    # Find the list of files to format
+    if cli.args.files:
+        cli.args.files = [os.path.join(os.environ['ORIG_CWD'], file) for file in cli.args.files]
+    else:
+        for dir in ['drivers', 'quantum', 'tests', 'tmk_core']:
+            for dirpath, dirnames, filenames in os.walk(dir):
+                if 'tmk_core/protocol/usb_hid' in dirpath:
+                    continue
+
+                for name in filenames:
+                    if name.endswith('.c') or name.endswith('.h') or name.endswith('.cpp'):
+                        cli.args.files.append(os.path.join(dirpath, name))
+
+    # Run clang-format on the files we've found
+    try:
+        subprocess.run(clang_format + cli.args.files, check=True)
+        cli.log.info('Successfully formatted the C code.')
+
+    except subprocess.CalledProcessError:
+        cli.log.error('Error formatting C code!')
+        return False
diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py
new file mode 100755
index 0000000000..234ffb12ca
--- /dev/null
+++ b/lib/python/qmk/cli/compile.py
@@ -0,0 +1,53 @@
+"""Compile a QMK Firmware.
+
+You can compile a keymap already in the repo or using a QMK Configurator export.
+"""
+import subprocess
+from argparse import FileType
+
+from milc import cli
+from qmk.commands import create_make_command
+from qmk.commands import parse_configurator_json
+from qmk.commands import compile_configurator_json
+
+import qmk.keymap
+import qmk.path
+
+
+@cli.argument('filename', nargs='?', arg_only=True, type=FileType('r'), help='The configurator export to compile')
+@cli.argument('-kb', '--keyboard', help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
+@cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Ignored when a configurator export is supplied.')
+@cli.subcommand('Compile a QMK Firmware.')
+def compile(cli):
+    """Compile a QMK Firmware.
+
+    If a Configurator export is supplied this command will create a new keymap, overwriting an existing keymap if one exists.
+
+    FIXME(skullydazed): add code to check and warn if the keymap already exists
+
+    If --keyboard and --keymap are provided this command will build a firmware based on that.
+
+    """
+    if cli.args.filename:
+        # Parse the configurator json
+        user_keymap = parse_configurator_json(cli.args.filename)
+
+        # Generate the keymap
+        keymap_path = qmk.path.keymap(user_keymap['keyboard'])
+        cli.log.info('Creating {fg_cyan}%s{style_reset_all} keymap in {fg_cyan}%s', user_keymap['keymap'], keymap_path)
+
+        # Compile the keymap
+        command = compile_configurator_json(cli.args.filename)
+
+        cli.log.info('Wrote keymap to {fg_cyan}%s/%s/keymap.c', keymap_path, user_keymap['keymap'])
+
+    elif cli.config.compile.keyboard and cli.config.compile.keymap:
+        # Generate the make command for a specific keyboard/keymap.
+        command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap)
+
+    else:
+        cli.log.error('You must supply a configurator export or both `--keyboard` and `--keymap`.')
+        return False
+
+    cli.log.info('Compiling keymap with {fg_cyan}%s\n\n', ' '.join(command))
+    subprocess.run(command)
diff --git a/lib/python/qmk/cli/config.py b/lib/python/qmk/cli/config.py
new file mode 100644
index 0000000000..e17d8bb9ba
--- /dev/null
+++ b/lib/python/qmk/cli/config.py
@@ -0,0 +1,116 @@
+"""Read and write configuration settings
+"""
+from milc import cli
+
+
+def print_config(section, key):
+    """Print a single config setting to stdout.
+    """
+    cli.echo('%s.%s{fg_cyan}={fg_reset}%s', section, key, cli.config[section][key])
+
+
+def show_config():
+    """Print the current configuration to stdout.
+    """
+    for section in cli.config:
+        for key in cli.config[section]:
+            print_config(section, key)
+
+
+def parse_config_token(config_token):
+    """Split a user-supplied configuration-token into its components.
+    """
+    section = option = value = None
+
+    if '=' in config_token and '.' not in config_token:
+        cli.log.error('Invalid configuration token, the key must be of the form <section>.<option>: %s', config_token)
+        return section, option, value
+
+    # Separate the key (<section>.<option>) from the value
+    if '=' in config_token:
+        key, value = config_token.split('=')
+    else:
+        key = config_token
+
+    # Extract the section and option from the key
+    if '.' in key:
+        section, option = key.split('.', 1)
+    else:
+        section = key
+
+    return section, option, value
+
+
+def set_config(section, option, value):
+    """Set a config key in the running config.
+    """
+    log_string = '%s.%s{fg_cyan}:{fg_reset} %s {fg_cyan}->{fg_reset} %s'
+    if cli.args.read_only:
+        log_string += ' {fg_red}(change not written)'
+
+    cli.echo(log_string, section, option, cli.config[section][option], value)
+
+    if not cli.args.read_only:
+        if value == 'None':
+            del cli.config[section][option]
+        else:
+            cli.config[section][option] = value
+
+
+@cli.argument('-ro', '--read-only', arg_only=True, action='store_true', help='Operate in read-only mode.')
+@cli.argument('configs', nargs='*', arg_only=True, help='Configuration options to read or write.')
+@cli.subcommand("Read and write configuration settings.")
+def config(cli):
+    """Read and write config settings.
+
+    This script iterates over the config_tokens supplied as argument. Each config_token has the following form:
+
+        section[.key][=value]
+
+    If only a section (EG 'compile') is supplied all keys for that section will be displayed.
+
+    If section.key is supplied the value for that single key will be displayed.
+
+    If section.key=value is supplied the value for that single key will be set.
+
+    If section.key=None is supplied the key will be deleted.
+
+    No validation is done to ensure that the supplied section.key is actually used by qmk scripts.
+    """
+    if not cli.args.configs:
+        return show_config()
+
+    # Process config_tokens
+    save_config = False
+
+    for argument in cli.args.configs:
+        # Split on space in case they quoted multiple config tokens
+        for config_token in argument.split(' '):
+            section, option, value = parse_config_token(config_token)
+
+            # Validation
+            if option and '.' in option:
+                cli.log.error('Config keys may not have more than one period! "%s" is not valid.', config_token)
+                return False
+
+            # Do what the user wants
+            if section and option and value:
+                # Write a configuration option
+                set_config(section, option, value)
+                if not cli.args.read_only:
+                    save_config = True
+
+            elif section and option:
+                # Display a single key
+                print_config(section, option)
+
+            elif section:
+                # Display an entire section
+                for key in cli.config[section]:
+                    print_config(section, key)
+
+    # Ending actions
+    if save_config:
+        cli.save_config()
+
+    return True
diff --git a/lib/python/qmk/cli/docs.py b/lib/python/qmk/cli/docs.py
new file mode 100644
index 0000000000..b419891396
--- /dev/null
+++ b/lib/python/qmk/cli/docs.py
@@ -0,0 +1,27 @@
+"""Serve QMK documentation locally
+"""
+import http.server
+
+from milc import cli
+
+
+class DocsHandler(http.server.SimpleHTTPRequestHandler):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, directory='docs', **kwargs)
+
+
+@cli.argument('-p', '--port', default=8936, type=int, help='Port number to use.')
+@cli.subcommand('Run a local webserver for QMK documentation.')
+def docs(cli):
+    """Spin up a local HTTPServer instance for the QMK docs.
+    """
+    with http.server.HTTPServer(('', cli.config.docs.port), DocsHandler) as httpd:
+        cli.log.info("Serving QMK docs at http://localhost:%d/", cli.config.docs.port)
+        cli.log.info("Press Control+C to exit.")
+
+        try:
+            httpd.serve_forever()
+        except KeyboardInterrupt:
+            cli.log.info("Stopping HTTP server...")
+        finally:
+            httpd.shutdown()
diff --git a/lib/python/qmk/cli/doctor.py b/lib/python/qmk/cli/doctor.py
new file mode 100755
index 0000000000..1010eafb33
--- /dev/null
+++ b/lib/python/qmk/cli/doctor.py
@@ -0,0 +1,75 @@
+"""QMK Python Doctor
+
+Check up for QMK environment.
+"""
+import platform
+import shutil
+import subprocess
+
+from milc import cli
+
+
+@cli.subcommand('Basic QMK environment checks')
+def doctor(cli):
+    """Basic QMK environment checks.
+
+    This is currently very simple, it just checks that all the expected binaries are on your system.
+
+    TODO(unclaimed):
+        * [ ] Compile a trivial program with each compiler
+        * [ ] Check for udev entries on linux
+    """
+    cli.log.info('QMK Doctor is checking your environment.')
+
+    # Make sure the basic CLI tools we need are available and can be executed.
+    binaries = ['dfu-programmer', 'avrdude', 'dfu-util', 'avr-gcc', 'arm-none-eabi-gcc', 'bin/qmk']
+    ok = True
+
+    for binary in binaries:
+        res = shutil.which(binary)
+        if res is None:
+            cli.log.error("{fg_red}QMK can't find %s in your path.", binary)
+            ok = False
+        else:
+            check = subprocess.run([binary, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=5)
+            if check.returncode in [0, 1]:
+                cli.log.info('Found {fg_cyan}%s', binary)
+            else:
+                cli.log.error("{fg_red}Can't run `%s --version`", binary)
+                ok = False
+
+    # Determine our OS and run platform specific tests
+    OS = platform.system()
+
+    if OS == "Darwin":
+        cli.log.info("Detected {fg_cyan}macOS.")
+
+    elif OS == "Linux":
+        cli.log.info("Detected {fg_cyan}Linux.")
+        if shutil.which('systemctl'):
+            mm_check = subprocess.run(['systemctl', 'list-unit-files'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=10, universal_newlines=True)
+            if mm_check.returncode == 0:
+                mm = False
+                for line in mm_check.stdout.split('\n'):
+                    if 'ModemManager' in line and 'enabled' in line:
+                        mm = True
+
+                if mm:
+                    cli.log.warn("{bg_yellow}Detected ModemManager. Please disable it if you are using a Pro-Micro.")
+
+            else:
+                cli.log.error('{bg_red}Could not run `systemctl list-unit-files`:')
+                cli.log.error(mm_check.stderr)
+
+        else:
+            cli.log.warn("Can't find systemctl to check for ModemManager.")
+
+    else:
+        cli.log.info("Assuming {fg_cyan}Windows.")
+
+    # Report a summary of our findings to the user
+    if ok:
+        cli.log.info('{fg_green}QMK is ready to go')
+    else:
+        cli.log.info('{fg_yellow}Problems detected, please fix these problems before proceeding.')
+        # FIXME(skullydazed): Link to a document about troubleshooting, or discord or something
diff --git a/lib/python/qmk/cli/flash.py b/lib/python/qmk/cli/flash.py
new file mode 100644
index 0000000000..031cb94967
--- /dev/null
+++ b/lib/python/qmk/cli/flash.py
@@ -0,0 +1,82 @@
+"""Compile and flash QMK Firmware
+
+You can compile a keymap already in the repo or using a QMK Configurator export.
+A bootloader must be specified.
+"""
+import subprocess
+
+import qmk.path
+from milc import cli
+from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json
+
+
+def print_bootloader_help():
+    """Prints the available bootloaders listed in docs.qmk.fm.
+    """
+    cli.log.info('Here are the available bootloaders:')
+    cli.echo('\tdfu')
+    cli.echo('\tdfu-ee')
+    cli.echo('\tdfu-split-left')
+    cli.echo('\tdfu-split-right')
+    cli.echo('\tavrdude')
+    cli.echo('\tBootloadHID')
+    cli.echo('\tdfu-util')
+    cli.echo('\tdfu-util-split-left')
+    cli.echo('\tdfu-util-split-right')
+    cli.echo('\tst-link-cli')
+    cli.echo('For more info, visit https://docs.qmk.fm/#/flashing')
+
+
+@cli.argument('-bl', '--bootloader', default='flash', help='The flash command, corresponding to qmk\'s make options of bootloaders.')
+@cli.argument('filename', nargs='?', arg_only=True, help='The configurator export JSON to compile. Use this if you dont want to specify a keymap and keyboard.')
+@cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.')
+@cli.argument('-kb', '--keyboard', help='The keyboard to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.')
+@cli.argument('-b', '--bootloaders', action='store_true', help='List the available bootloaders.')
+@cli.subcommand('QMK Flash.')
+def flash(cli):
+    """Compile and or flash QMK Firmware or keyboard/layout
+
+    If a Configurator JSON export is supplied this command will create a new keymap. Keymap and Keyboard arguments
+    will be ignored.
+
+    If no file is supplied, keymap and keyboard are expected.
+
+    If bootloader is omitted, the one according to the rules.mk will be used.
+
+    """
+    command = []
+    if cli.args.bootloaders:
+        # Provide usage and list bootloaders
+        cli.echo('usage: qmk flash [-h] [-b] [-kb KEYBOARD] [-km KEYMAP] [-bl BOOTLOADER] [filename]')
+        print_bootloader_help()
+        return False
+
+    elif cli.args.keymap and not cli.args.keyboard:
+        # If only a keymap was given but no keyboard, suggest listing keyboards
+        cli.echo('usage: qmk flash [-h] [-b] [-kb KEYBOARD] [-km KEYMAP] [-bl BOOTLOADER] [filename]')
+        cli.log.error('run \'qmk list_keyboards\' to find out the supported keyboards')
+        return False
+
+    elif cli.args.filename:
+        # Get keymap path to log info
+        user_keymap = parse_configurator_json(cli.args.filename)
+        keymap_path = qmk.path.keymap(user_keymap['keyboard'])
+
+        cli.log.info('Creating {fg_cyan}%s{style_reset_all} keymap in {fg_cyan}%s', user_keymap['keymap'], keymap_path)
+
+        # Convert the JSON into a C file and write it to disk.
+        command = compile_configurator_json(cli.args.filename, cli.args.bootloader)
+
+        cli.log.info('Wrote keymap to {fg_cyan}%s/%s/keymap.c', keymap_path, user_keymap['keymap'])
+
+    elif cli.args.keyboard and cli.args.keymap:
+        # Generate the make command for a specific keyboard/keymap.
+        command = create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, cli.args.bootloader)
+
+    else:
+        cli.echo('usage: qmk flash [-h] [-b] [-kb KEYBOARD] [-km KEYMAP] [-bl BOOTLOADER] [filename]')
+        cli.log.error('You must supply a configurator export or both `--keyboard` and `--keymap`. You can also specify a bootloader with --bootloader. Use --bootloaders to list the available bootloaders.')
+        return False
+
+    cli.log.info('Flashing keymap with {fg_cyan}%s\n\n', ' '.join(command))
+    subprocess.run(command)
diff --git a/lib/python/qmk/cli/hello.py b/lib/python/qmk/cli/hello.py
new file mode 100755
index 0000000000..bee28c3013
--- /dev/null
+++ b/lib/python/qmk/cli/hello.py
@@ -0,0 +1,13 @@
+"""QMK Python Hello World
+
+This is an example QMK CLI script.
+"""
+from milc import cli
+
+
+@cli.argument('-n', '--name', default='World', help='Name to greet.')
+@cli.subcommand('QMK Hello World.')
+def hello(cli):
+    """Log a friendly greeting.
+    """
+    cli.log.info('Hello, %s!', cli.config.hello.name)
diff --git a/lib/python/qmk/cli/json/__init__.py b/lib/python/qmk/cli/json/__init__.py
new file mode 100644
index 0000000000..f4ebfc45b4
--- /dev/null
+++ b/lib/python/qmk/cli/json/__init__.py
@@ -0,0 +1,5 @@
+"""QMK CLI JSON Subcommands
+
+We list each subcommand here explicitly because all the reliable ways of searching for modules are slow and delay startup.
+"""
+from . import keymap
diff --git a/lib/python/qmk/cli/json/keymap.py b/lib/python/qmk/cli/json/keymap.py
new file mode 100755
index 0000000000..a030ab53d6
--- /dev/null
+++ b/lib/python/qmk/cli/json/keymap.py
@@ -0,0 +1,55 @@
+"""Generate a keymap.c from a configurator export.
+"""
+import json
+import os
+
+from milc import cli
+
+import qmk.keymap
+
+
+@cli.argument('-o', '--output', arg_only=True, help='File to write to')
+@cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages")
+@cli.argument('filename', arg_only=True, help='Configurator JSON file')
+@cli.subcommand('Creates a keymap.c from a QMK Configurator export.')
+def json_keymap(cli):
+    """Generate a keymap.c from a configurator export.
+
+    This command uses the `qmk.keymap` module to generate a keymap.c from a configurator export. The generated keymap is written to stdout, or to a file if -o is provided.
+    """
+    # Error checking
+    if cli.args.filename == ('-'):
+        cli.log.error('Reading from STDIN is not (yet) supported.')
+        cli.print_usage()
+        exit(1)
+    if not os.path.exists(qmk.path.normpath(cli.args.filename)):
+        cli.log.error('JSON file does not exist!')
+        cli.print_usage()
+        exit(1)
+
+    # Environment processing
+    if cli.args.output == ('-'):
+        cli.args.output = None
+
+    # Parse the configurator json
+    with open(qmk.path.normpath(cli.args.filename), 'r') as fd:
+        user_keymap = json.load(fd)
+
+    # Generate the keymap
+    keymap_c = qmk.keymap.generate(user_keymap['keyboard'], user_keymap['layout'], user_keymap['layers'])
+
+    if cli.args.output:
+        output_dir = os.path.dirname(cli.args.output)
+
+        if not os.path.exists(output_dir):
+            os.makedirs(output_dir)
+
+        output_file = qmk.path.normpath(cli.args.output)
+        with open(output_file, 'w') as keymap_fd:
+            keymap_fd.write(keymap_c)
+
+        if not cli.args.quiet:
+            cli.log.info('Wrote keymap to %s.', cli.args.output)
+
+    else:
+        print(keymap_c)
diff --git a/lib/python/qmk/cli/kle2json.py b/lib/python/qmk/cli/kle2json.py
new file mode 100755
index 0000000000..00f63d3622
--- /dev/null
+++ b/lib/python/qmk/cli/kle2json.py
@@ -0,0 +1,75 @@
+"""Convert raw KLE to JSON
+"""
+import json
+import os
+from pathlib import Path
+from decimal import Decimal
+from collections import OrderedDict
+
+from milc import cli
+from kle2xy import KLE2xy
+
+from qmk.converter import kle2qmk
+
+
+class CustomJSONEncoder(json.JSONEncoder):
+    def default(self, obj):
+        try:
+            if isinstance(obj, Decimal):
+                if obj % 2 in (Decimal(0), Decimal(1)):
+                    return int(obj)
+                return float(obj)
+        except TypeError:
+            pass
+        return json.JSONEncoder.default(self, obj)
+
+
+@cli.argument('filename', help='The KLE raw txt to convert')
+@cli.argument('-f', '--force', action='store_true', help='Flag to overwrite current info.json')
+@cli.subcommand('Convert a KLE layout to a Configurator JSON')
+def kle2json(cli):
+    """Convert a KLE layout to QMK's layout format.
+    """  # If filename is a path
+    if cli.args.filename.startswith("/") or cli.args.filename.startswith("./"):
+        file_path = Path(cli.args.filename)
+    # Otherwise assume it is a file name
+    else:
+        file_path = Path(os.environ['ORIG_CWD'], cli.args.filename)
+    # Check for valid file_path for more graceful failure
+    if not file_path.exists():
+        return cli.log.error('File {fg_cyan}%s{style_reset_all} was not found.', str(file_path))
+    out_path = file_path.parent
+    raw_code = file_path.open().read()
+    # Check if info.json exists, allow overwrite with force
+    if Path(out_path, "info.json").exists() and not cli.args.force:
+        cli.log.error('File {fg_cyan}%s/info.json{style_reset_all} already exists, use -f or --force to overwrite.', str(out_path))
+        return False
+    try:
+        # Convert KLE raw to x/y coordinates (using kle2xy package from skullydazed)
+        kle = KLE2xy(raw_code)
+    except Exception as e:
+        cli.log.error('Could not parse KLE raw data: %s', raw_code)
+        cli.log.exception(e)
+        # FIXME: This should be better
+        return cli.log.error('Could not parse KLE raw data.')
+    keyboard = OrderedDict(
+        keyboard_name=kle.name,
+        url='',
+        maintainer='qmk',
+        width=kle.columns,
+        height=kle.rows,
+        layouts={'LAYOUT': {
+            'layout': 'LAYOUT_JSON_HERE'
+        }},
+    )
+    # Initialize keyboard with json encoded from ordered dict
+    keyboard = json.dumps(keyboard, indent=4, separators=(', ', ': '), sort_keys=False, cls=CustomJSONEncoder)
+    # Initialize layout with kle2qmk from converter module
+    layout = json.dumps(kle2qmk(kle), separators=(', ', ':'), cls=CustomJSONEncoder)
+    # Replace layout in keyboard json
+    keyboard = keyboard.replace('"LAYOUT_JSON_HERE"', layout)
+    # Write our info.json
+    file = open(str(out_path) + "/info.json", "w")
+    file.write(keyboard)
+    file.close()
+    cli.log.info('Wrote out {fg_cyan}%s/info.json', str(out_path))
diff --git a/lib/python/qmk/cli/list/__init__.py b/lib/python/qmk/cli/list/__init__.py
new file mode 100644
index 0000000000..c36ba69548
--- /dev/null
+++ b/lib/python/qmk/cli/list/__init__.py
@@ -0,0 +1 @@
+from . import keyboards
diff --git a/lib/python/qmk/cli/list/keyboards.py b/lib/python/qmk/cli/list/keyboards.py
new file mode 100644
index 0000000000..76e7760e84
--- /dev/null
+++ b/lib/python/qmk/cli/list/keyboards.py
@@ -0,0 +1,27 @@
+"""List the keyboards currently defined within QMK
+"""
+import os
+import glob
+
+from milc import cli
+
+BASE_PATH = os.path.join(os.getcwd(), "keyboards") + os.path.sep
+KB_WILDCARD = os.path.join(BASE_PATH, "**", "rules.mk")
+
+
+def find_name(path):
+    """Determine the keyboard name by stripping off the base_path and rules.mk.
+    """
+    return path.replace(BASE_PATH, "").replace(os.path.sep + "rules.mk", "")
+
+
+@cli.subcommand("List the keyboards currently defined within QMK")
+def list_keyboards(cli):
+    """List the keyboards currently defined within QMK
+    """
+    # find everywhere we have rules.mk where keymaps isn't in the path
+    paths = [path for path in glob.iglob(KB_WILDCARD, recursive=True) if 'keymaps' not in path]
+
+    # Extract the keyboard name from the path and print it
+    for keyboard_name in sorted(map(find_name, paths)):
+        print(keyboard_name)
diff --git a/lib/python/qmk/cli/new/__init__.py b/lib/python/qmk/cli/new/__init__.py
new file mode 100644
index 0000000000..c6a26939b8
--- /dev/null
+++ b/lib/python/qmk/cli/new/__init__.py
@@ -0,0 +1 @@
+from . import keymap
diff --git a/lib/python/qmk/cli/new/keymap.py b/lib/python/qmk/cli/new/keymap.py
new file mode 100755
index 0000000000..96525e28e1
--- /dev/null
+++ b/lib/python/qmk/cli/new/keymap.py
@@ -0,0 +1,40 @@
+"""This script automates the copying of the default keymap into your own keymap.
+"""
+import os
+import shutil
+
+from milc import cli
+
+
+@cli.argument('-kb', '--keyboard', help='Specify keyboard name. Example: 1upkeyboards/1up60hse')
+@cli.argument('-km', '--keymap', help='Specify the name for the new keymap directory')
+@cli.subcommand('Creates a new keymap for the keyboard of your choosing')
+def new_keymap(cli):
+    """Creates a new keymap for the keyboard of your choosing.
+    """
+    # ask for user input if keyboard or keymap was not provided in the command line
+    keyboard = cli.config.new_keymap.keyboard if cli.config.new_keymap.keyboard else input("Keyboard Name: ")
+    keymap = cli.config.new_keymap.keymap if cli.config.new_keymap.keymap else input("Keymap Name: ")
+
+    # generate keymap paths
+    kb_path = os.path.join(os.getcwd(), "keyboards", keyboard)
+    keymap_path_default = os.path.join(kb_path, "keymaps/default")
+    keymap_path = os.path.join(kb_path, "keymaps/%s" % keymap)
+
+    # check directories
+    if not os.path.exists(kb_path):
+        cli.log.error('Keyboard %s does not exist!', kb_path)
+        exit(1)
+    if not os.path.exists(keymap_path_default):
+        cli.log.error('Keyboard default %s does not exist!', keymap_path_default)
+        exit(1)
+    if os.path.exists(keymap_path):
+        cli.log.error('Keymap %s already exists!', keymap_path)
+        exit(1)
+
+    # create user directory with default keymap files
+    shutil.copytree(keymap_path_default, keymap_path, symlinks=True)
+
+    # end message to user
+    cli.log.info("%s keymap directory created in: %s", keymap, keymap_path)
+    cli.log.info("Compile a firmware with your new keymap by typing: \n" + "qmk compile -kb %s -km %s", keyboard, keymap)
diff --git a/lib/python/qmk/cli/pyformat.py b/lib/python/qmk/cli/pyformat.py
new file mode 100755
index 0000000000..a53ba40c0a
--- /dev/null
+++ b/lib/python/qmk/cli/pyformat.py
@@ -0,0 +1,17 @@
+"""Format python code according to QMK's style.
+"""
+from milc import cli
+
+import subprocess
+
+
+@cli.subcommand("Format python code according to QMK's style.")
+def pyformat(cli):
+    """Format python code according to QMK's style.
+    """
+    try:
+        subprocess.run(['yapf', '-vv', '-ri', 'bin/qmk', 'lib/python'], check=True)
+        cli.log.info('Successfully formatted the python code in `bin/qmk` and `lib/python`.')
+
+    except subprocess.CalledProcessError:
+        cli.log.error('Error formatting python code!')
diff --git a/lib/python/qmk/cli/pytest.py b/lib/python/qmk/cli/pytest.py
new file mode 100644
index 0000000000..09611d750f
--- /dev/null
+++ b/lib/python/qmk/cli/pytest.py
@@ -0,0 +1,16 @@
+"""QMK Python Unit Tests
+
+QMK script to run unit and integration tests against our python code.
+"""
+import subprocess
+
+from milc import cli
+
+
+@cli.subcommand('QMK Python Unit Tests')
+def pytest(cli):
+    """Run several linting/testing commands.
+    """
+    flake8 = subprocess.run(['flake8', 'lib/python', 'bin/qmk'])
+    nose2 = subprocess.run(['nose2', '-v'])
+    return flake8.returncode | nose2.returncode
diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py
new file mode 100644
index 0000000000..f83a89578e
--- /dev/null
+++ b/lib/python/qmk/commands.py
@@ -0,0 +1,59 @@
+"""Functions that build make commands
+"""
+import json
+import qmk.keymap
+
+
+def create_make_command(keyboard, keymap, target=None):
+    """Create a make compile command
+
+    Args:
+        keyboard
+            The path of the keyboard, for example 'plank'
+
+        keymap
+            The name of the keymap, for example 'algernon'
+
+        target
+            Usually a bootloader.
+
+    Returns:
+        A command that can be run to make the specified keyboard and keymap
+    """
+    if target is None:
+        return ['make', ':'.join((keyboard, keymap))]
+    return ['make', ':'.join((keyboard, keymap, target))]
+
+
+def parse_configurator_json(configurator_filename):
+    """Open and parse a configurator json export
+    """
+    file = open(configurator_filename)
+    user_keymap = json.load(file)
+    file.close()
+    return user_keymap
+
+
+def compile_configurator_json(configurator_filename, bootloader=None):
+    """Convert a configurator export JSON file into a C file
+
+    Args:
+        configurator_filename
+            The configurator JSON export file
+
+        bootloader
+            A bootloader to flash
+
+    Returns:
+        A command to run to compile and flash the C file.
+    """
+    # Parse the configurator json
+    user_keymap = parse_configurator_json(configurator_filename)
+
+    # Write the keymap C file
+    qmk.keymap.write(user_keymap['keyboard'], user_keymap['keymap'], user_keymap['layout'], user_keymap['layers'])
+
+    # Return a command that can be run to make the keymap and flash if given
+    if bootloader is None:
+        return create_make_command(user_keymap['keyboard'], user_keymap['keymap'])
+    return create_make_command(user_keymap['keyboard'], user_keymap['keymap'], bootloader)
diff --git a/lib/python/qmk/converter.py b/lib/python/qmk/converter.py
new file mode 100644
index 0000000000..bbd3531317
--- /dev/null
+++ b/lib/python/qmk/converter.py
@@ -0,0 +1,33 @@
+"""Functions to convert to and from QMK formats
+"""
+from collections import OrderedDict
+
+
+def kle2qmk(kle):
+    """Convert a KLE layout to QMK's layout format.
+    """
+    layout = []
+
+    for row in kle:
+        for key in row:
+            if key['decal']:
+                continue
+
+            qmk_key = OrderedDict(
+                label="",
+                x=key['column'],
+                y=key['row'],
+            )
+
+            if key['width'] != 1:
+                qmk_key['w'] = key['width']
+            if key['height'] != 1:
+                qmk_key['h'] = key['height']
+            if 'name' in key and key['name']:
+                qmk_key['label'] = key['name'].split('\n', 1)[0]
+            else:
+                del (qmk_key['label'])
+
+            layout.append(qmk_key)
+
+    return layout
diff --git a/lib/python/qmk/errors.py b/lib/python/qmk/errors.py
new file mode 100644
index 0000000000..4a8a91556b
--- /dev/null
+++ b/lib/python/qmk/errors.py
@@ -0,0 +1,5 @@
+class NoSuchKeyboardError(Exception):
+    """Raised when we can't find a keyboard/keymap directory.
+    """
+    def __init__(self, message):
+        self.message = message
diff --git a/lib/python/qmk/keymap.py b/lib/python/qmk/keymap.py
new file mode 100644
index 0000000000..f4a2893f38
--- /dev/null
+++ b/lib/python/qmk/keymap.py
@@ -0,0 +1,96 @@
+"""Functions that help you work with QMK keymaps.
+"""
+import os
+
+import qmk.path
+
+# The `keymap.c` template to use when a keyboard doesn't have its own
+DEFAULT_KEYMAP_C = """#include QMK_KEYBOARD_H
+
+/* THIS FILE WAS GENERATED!
+ *
+ * This file was generated by qmk-compile-json. You may or may not want to
+ * edit it directly.
+ */
+
+const uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = {
+__KEYMAP_GOES_HERE__
+};
+"""
+
+
+def template(keyboard):
+    """Returns the `keymap.c` template for a keyboard.
+
+    If a template exists in `keyboards/<keyboard>/templates/keymap.c` that
+    text will be used instead of `DEFAULT_KEYMAP_C`.
+
+    Args:
+        keyboard
+            The keyboard to return a template for.
+    """
+    template_name = 'keyboards/%s/templates/keymap.c' % keyboard
+
+    if os.path.exists(template_name):
+        with open(template_name, 'r') as fd:
+            return fd.read()
+
+    return DEFAULT_KEYMAP_C
+
+
+def generate(keyboard, layout, layers):
+    """Returns a keymap.c for the specified keyboard, layout, and layers.
+
+    Args:
+        keyboard
+            The name of the keyboard
+
+        layout
+            The LAYOUT macro this keymap uses.
+
+        layers
+            An array of arrays describing the keymap. Each item in the inner array should be a string that is a valid QMK keycode.
+    """
+    layer_txt = []
+    for layer_num, layer in enumerate(layers):
+        if layer_num != 0:
+            layer_txt[-1] = layer_txt[-1] + ','
+        layer_keys = ', '.join(layer)
+        layer_txt.append('\t[%s] = %s(%s)' % (layer_num, layout, layer_keys))
+
+    keymap = '\n'.join(layer_txt)
+    keymap_c = template(keyboard)
+
+    return keymap_c.replace('__KEYMAP_GOES_HERE__', keymap)
+
+
+def write(keyboard, keymap, layout, layers):
+    """Generate the `keymap.c` and write it to disk.
+
+    Returns the filename written to.
+
+    Args:
+        keyboard
+            The name of the keyboard
+
+        keymap
+            The name of the keymap
+
+        layout
+            The LAYOUT macro this keymap uses.
+
+        layers
+            An array of arrays describing the keymap. Each item in the inner array should be a string that is a valid QMK keycode.
+    """
+    keymap_c = generate(keyboard, layout, layers)
+    keymap_path = qmk.path.keymap(keyboard)
+    keymap_dir = os.path.join(keymap_path, keymap)
+    keymap_file = os.path.join(keymap_dir, 'keymap.c')
+
+    if not os.path.exists(keymap_dir):
+        os.makedirs(keymap_dir)
+
+    with open(keymap_file, 'w') as keymap_fd:
+        keymap_fd.write(keymap_c)
+
+    return keymap_file
diff --git a/lib/python/qmk/path.py b/lib/python/qmk/path.py
new file mode 100644
index 0000000000..cf087265fb
--- /dev/null
+++ b/lib/python/qmk/path.py
@@ -0,0 +1,35 @@
+"""Functions that help us work with files and folders.
+"""
+import logging
+import os
+
+from qmk.errors import NoSuchKeyboardError
+
+
+def keymap(keyboard):
+    """Locate the correct directory for storing a keymap.
+
+    Args:
+        keyboard
+            The name of the keyboard. Example: clueboard/66/rev3
+    """
+    for directory in ['.', '..', '../..', '../../..', '../../../..', '../../../../..']:
+        basepath = os.path.normpath(os.path.join('keyboards', keyboard, directory, 'keymaps'))
+
+        if os.path.exists(basepath):
+            return basepath
+
+    logging.error('Could not find keymaps directory!')
+    raise NoSuchKeyboardError('Could not find keymaps directory for: %s' % keyboard)
+
+
+def normpath(path):
+    """Returns the fully resolved absolute path to a file.
+
+    This function will return the absolute path to a file as seen from the
+    directory the script was called from.
+    """
+    if path and path[0] == '/':
+        return os.path.normpath(path)
+
+    return os.path.normpath(os.path.join(os.environ['ORIG_CWD'], path))
diff --git a/lib/python/qmk/tests/__init__.py b/lib/python/qmk/tests/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/lib/python/qmk/tests/__init__.py
diff --git a/lib/python/qmk/tests/attrdict.py b/lib/python/qmk/tests/attrdict.py
new file mode 100644
index 0000000000..a2584b9233
--- /dev/null
+++ b/lib/python/qmk/tests/attrdict.py
@@ -0,0 +1,8 @@
+class AttrDict(dict):
+    """A dictionary that can be accessed by attributes.
+
+    This should only be used to mock objects for unit testing. Please do not use this outside of qmk.tests.
+    """
+    def __init__(self, *args, **kwargs):
+        super(AttrDict, self).__init__(*args, **kwargs)
+        self.__dict__ = self
diff --git a/lib/python/qmk/tests/kle.txt b/lib/python/qmk/tests/kle.txt
new file mode 100644
index 0000000000..862a899ab9
--- /dev/null
+++ b/lib/python/qmk/tests/kle.txt
@@ -0,0 +1,5 @@
+["¬\n`","!\n1","\"\n2","£\n3","$\n4","%\n5","^\n6","&\n7","*\n8","(\n9",")\n0","_\n-","+\n=",{w:2},"Backspace"],
+[{w:1.5},"Tab","Q","W","E","R","T","Y","U","I","O","P","{\n[","}\n]",{x:0.25,w:1.25,h:2,w2:1.5,h2:1,x2:-0.25},"Enter"],
+[{w:1.75},"Caps Lock","A","S","D","F","G","H","J","K","L",":\n;","@\n'","~\n#"],
+[{w:1.25},"Shift","|\n\\","Z","X","C","V","B","N","M","<\n,",">\n.","?\n/",{w:2.75},"Shift"],
+[{w:1.25},"Ctrl",{w:1.25},"Win",{w:1.25},"Alt",{a:7,w:6.25},"",{a:4,w:1.25},"AltGr",{w:1.25},"Win",{w:1.25},"Menu",{w:1.25},"Ctrl"]
diff --git a/lib/python/qmk/tests/onekey_export.json b/lib/python/qmk/tests/onekey_export.json
new file mode 100644
index 0000000000..95f0a980fe
--- /dev/null
+++ b/lib/python/qmk/tests/onekey_export.json
@@ -0,0 +1,6 @@
+{
+    "keyboard":"handwired/onekey/pytest",
+    "keymap":"pytest_unittest",
+    "layout":"LAYOUT",
+    "layers":[["KC_A"]]
+}
diff --git a/lib/python/qmk/tests/test_cli_commands.py b/lib/python/qmk/tests/test_cli_commands.py
new file mode 100644
index 0000000000..3f75cef3e1
--- /dev/null
+++ b/lib/python/qmk/tests/test_cli_commands.py
@@ -0,0 +1,56 @@
+import subprocess
+
+
+def check_subcommand(command, *args):
+    cmd = ['bin/qmk', command] + list(args)
+    return subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
+
+
+def test_cformat():
+    assert check_subcommand('cformat', 'tmk_core/common/keyboard.c').returncode == 0
+
+
+def test_compile():
+    assert check_subcommand('compile', '-kb', 'handwired/onekey/pytest', '-km', 'default').returncode == 0
+
+
+def test_flash():
+    assert check_subcommand('flash', '-b').returncode == 1
+    assert check_subcommand('flash').returncode == 1
+
+
+def test_config():
+    result = check_subcommand('config')
+    assert result.returncode == 0
+    assert 'general.color' in result.stdout
+
+
+def test_kle2json():
+    assert check_subcommand('kle2json', 'kle.txt', '-f').returncode == 0
+
+
+def test_doctor():
+    result = check_subcommand('doctor')
+    assert result.returncode == 0
+    assert 'QMK Doctor is checking your environment.' in result.stderr
+    assert 'QMK is ready to go' in result.stderr
+
+
+def test_hello():
+    result = check_subcommand('hello')
+    assert result.returncode == 0
+    assert 'Hello,' in result.stderr
+
+
+def test_pyformat():
+    result = check_subcommand('pyformat')
+    assert result.returncode == 0
+    assert 'Successfully formatted the python code' in result.stderr
+
+
+def test_list_keyboards():
+    result = check_subcommand('list-keyboards')
+    assert result.returncode == 0
+    # check to see if a known keyboard is returned
+    # this will fail if handwired/onekey/pytest is removed
+    assert 'handwired/onekey/pytest' in result.stdout
diff --git a/lib/python/qmk/tests/test_qmk_errors.py b/lib/python/qmk/tests/test_qmk_errors.py
new file mode 100644
index 0000000000..1d8690b7ef
--- /dev/null
+++ b/lib/python/qmk/tests/test_qmk_errors.py
@@ -0,0 +1,8 @@
+from qmk.errors import NoSuchKeyboardError
+
+
+def test_NoSuchKeyboardError():
+    try:
+        raise NoSuchKeyboardError("test message")
+    except NoSuchKeyboardError as e:
+        assert e.message == 'test message'
diff --git a/lib/python/qmk/tests/test_qmk_keymap.py b/lib/python/qmk/tests/test_qmk_keymap.py
new file mode 100644
index 0000000000..2db625600e
--- /dev/null
+++ b/lib/python/qmk/tests/test_qmk_keymap.py
@@ -0,0 +1,19 @@
+import qmk.keymap
+
+
+def test_template_onekey_proton_c():
+    templ = qmk.keymap.template('handwired/onekey/proton_c')
+    assert templ == qmk.keymap.DEFAULT_KEYMAP_C
+
+
+def test_template_onekey_pytest():
+    templ = qmk.keymap.template('handwired/onekey/pytest')
+    assert templ == 'const uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = {__KEYMAP_GOES_HERE__};\n'
+
+
+def test_generate_onekey_pytest():
+    templ = qmk.keymap.generate('handwired/onekey/pytest', 'LAYOUT', [['KC_A']])
+    assert templ == 'const uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = {	[0] = LAYOUT(KC_A)};\n'
+
+
+# FIXME(skullydazed): Add a test for qmk.keymap.write that mocks up an FD.
diff --git a/lib/python/qmk/tests/test_qmk_path.py b/lib/python/qmk/tests/test_qmk_path.py
new file mode 100644
index 0000000000..d6961a0f65
--- /dev/null
+++ b/lib/python/qmk/tests/test_qmk_path.py
@@ -0,0 +1,13 @@
+import os
+
+import qmk.path
+
+
+def test_keymap_onekey_pytest():
+    path = qmk.path.keymap('handwired/onekey/pytest')
+    assert path == 'keyboards/handwired/onekey/keymaps'
+
+
+def test_normpath():
+    path = qmk.path.normpath('lib/python')
+    assert path == os.path.join(os.environ['ORIG_CWD'], 'lib/python')