Add support for out-of-tree userspace

Makefile

@@ -352,7 +352,21 @@ define PARSE_KEYBOARD
     LAYOUT_KEYMAPS :=
     $$(foreach LAYOUT,$$(KEYBOARD_LAYOUTS),$$(eval LAYOUT_KEYMAPS += $$(notdir $$(patsubst %/.,%,$$(wildcard $(ROOT_DIR)/layouts/*/$$(LAYOUT)/*/.)))))
 
-    KEYMAPS := $$(sort $$(KEYMAPS) $$(LAYOUT_KEYMAPS))
+    # Look for out-of-tree keymaps
+    # Both keyboard- and layout-specific keymaps are supported
+    ifneq ("$(EXTERNAL_USERSPACE)", "")
+        EXT_KEYMAPS :=
+        EXT_KEYMAPS += $$(notdir $$(patsubst %/.,%,$$(wildcard $(EXTERNAL_USERSPACE)/keyboards/$$(KEYBOARD_FOLDER_PATH_1)/*/.)))
+        EXT_KEYMAPS += $$(notdir $$(patsubst %/.,%,$$(wildcard $(EXTERNAL_USERSPACE)/keyboards/$$(KEYBOARD_FOLDER_PATH_2)/*/.)))
+        EXT_KEYMAPS += $$(notdir $$(patsubst %/.,%,$$(wildcard $(EXTERNAL_USERSPACE)/keyboards/$$(KEYBOARD_FOLDER_PATH_3)/*/.)))
+        EXT_KEYMAPS += $$(notdir $$(patsubst %/.,%,$$(wildcard $(EXTERNAL_USERSPACE)/keyboards/$$(KEYBOARD_FOLDER_PATH_4)/*/.)))
+        EXT_KEYMAPS += $$(notdir $$(patsubst %/.,%,$$(wildcard $(EXTERNAL_USERSPACE)/keyboards/$$(KEYBOARD_FOLDER_PATH_5)/*/.)))
+
+        EXT_LAYOUT_KEYMAPS :=
+        $$(foreach LAYOUT,$$(KEYBOARD_LAYOUTS),$$(eval EXT_LAYOUT_KEYMAPS += $$(notdir $$(patsubst %/.,%,$$(wildcard $(EXTERNAL_USERSPACE)/layouts/$$(LAYOUT)/*/.)))))
+    endif
+
+    KEYMAPS := $$(sort $$(KEYMAPS) $$(LAYOUT_KEYMAPS) $$(EXT_KEYMAPS) $$(EXT_LAYOUT_KEYMAPS))
 
     # if the rule after removing the start of it is empty (we haven't specified a kemap or target)
     # compile all the keymaps

build_json.mk

@@ -1,29 +1,60 @@
-# Look for a json keymap file
-ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_5)/keymap.json)","")
-    KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
-    KEYMAP_JSON := $(MAIN_KEYMAP_PATH_5)/keymap.json
-    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_5)
-else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_4)/keymap.json)","")
-    KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
-    KEYMAP_JSON := $(MAIN_KEYMAP_PATH_4)/keymap.json
-    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_4)
-else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_3)/keymap.json)","")
-    KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
-    KEYMAP_JSON := $(MAIN_KEYMAP_PATH_3)/keymap.json
-    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_3)
-else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_2)/keymap.json)","")
-    KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
-    KEYMAP_JSON := $(MAIN_KEYMAP_PATH_2)/keymap.json
-    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_2)
-else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_1)/keymap.json)","")
-    KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
-    KEYMAP_JSON := $(MAIN_KEYMAP_PATH_1)/keymap.json
-    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_1)
+ifneq ($(EXTERNAL_USERSPACE), )
+    # Look for out-of-tree json keymap file
+    ifneq ("$(wildcard $(EXTERNAL_USERSPACE)/keyboards/$(KEYBOARD_FOLDER_PATH_5)/$(KEYMAP)/keymap.json)", "")
+        KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
+        KEYMAP_JSON := $(EXTERNAL_USERSPACE)/keyboards/$(KEYBOARD_FOLDER_PATH_5)/$(KEYMAP)/keymap.json
+        KEYMAP_PATH := $(EXTERNAL_USERSPACE)/keyboards/$(KEYBOARD_FOLDER_PATH_5)/$(KEYMAP)
+    else ifneq ("$(wildcard $(EXTERNAL_USERSPACE)/keyboards/$(KEYBOARD_FOLDER_PATH_4)/$(KEYMAP)/keymap.json)", "")
+        KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
+        KEYMAP_JSON := $(EXTERNAL_USERSPACE)/keyboards/$(KEYBOARD_FOLDER_PATH_4)/$(KEYMAP)/keymap.json
+        KEYMAP_PATH := $(EXTERNAL_USERSPACE)/keyboards/$(KEYBOARD_FOLDER_PATH_4)/$(KEYMAP)
+    else ifneq ("$(wildcard $(EXTERNAL_USERSPACE)/keyboards/$(KEYBOARD_FOLDER_PATH_3)/$(KEYMAP)/keymap.json)", "")
+        KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
+        KEYMAP_JSON := $(EXTERNAL_USERSPACE)/keyboards/$(KEYBOARD_FOLDER_PATH_3)/$(KEYMAP)/keymap.json
+        KEYMAP_PATH := $(EXTERNAL_USERSPACE)/keyboards/$(KEYBOARD_FOLDER_PATH_3)/$(KEYMAP)
+    else ifneq ("$(wildcard $(EXTERNAL_USERSPACE)/keyboards/$(KEYBOARD_FOLDER_PATH_2)/$(KEYMAP)/keymap.json)", "")
+        KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
+        KEYMAP_JSON := $(EXTERNAL_USERSPACE)/keyboards/$(KEYBOARD_FOLDER_PATH_2)/$(KEYMAP)/keymap.json
+        KEYMAP_PATH := $(EXTERNAL_USERSPACE)/keyboards/$(KEYBOARD_FOLDER_PATH_2)/$(KEYMAP)
+    else ifneq ("$(wildcard $(EXTERNAL_USERSPACE)/keyboards/$(KEYBOARD_FOLDER_PATH_1)/$(KEYMAP)/keymap.json)", "")
+        KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
+        KEYMAP_JSON := $(EXTERNAL_USERSPACE)/keyboards/$(KEYBOARD_FOLDER_PATH_1)/$(KEYMAP)/keymap.json
+        KEYMAP_PATH := $(EXTERNAL_USERSPACE)/keyboards/$(KEYBOARD_FOLDER_PATH_1)/$(KEYMAP)
+    endif
+endif
+
+ifeq ("$(wildcard $(KEYMAP_PATH))", "")
+    # Look for in-tree json keymap file
+    ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_5)/keymap.json)","")
+        KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
+        KEYMAP_JSON := $(MAIN_KEYMAP_PATH_5)/keymap.json
+        KEYMAP_PATH := $(MAIN_KEYMAP_PATH_5)
+    else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_4)/keymap.json)","")
+        KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
+        KEYMAP_JSON := $(MAIN_KEYMAP_PATH_4)/keymap.json
+        KEYMAP_PATH := $(MAIN_KEYMAP_PATH_4)
+    else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_3)/keymap.json)","")
+        KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
+        KEYMAP_JSON := $(MAIN_KEYMAP_PATH_3)/keymap.json
+        KEYMAP_PATH := $(MAIN_KEYMAP_PATH_3)
+    else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_2)/keymap.json)","")
+        KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
+        KEYMAP_JSON := $(MAIN_KEYMAP_PATH_2)/keymap.json
+        KEYMAP_PATH := $(MAIN_KEYMAP_PATH_2)
+    else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_1)/keymap.json)","")
+        KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
+        KEYMAP_JSON := $(MAIN_KEYMAP_PATH_1)/keymap.json
+        KEYMAP_PATH := $(MAIN_KEYMAP_PATH_1)
+    endif
 endif
 
 # Load the keymap-level rules.mk if exists
 ifneq ("$(wildcard $(KEYMAP_PATH))", "")
     -include $(KEYMAP_PATH)/rules.mk
+    # If EXT_SRC exists, add all files to SRC with the EXT_KM_PATH prefix so make can find it
+    ifneq ($(EXT_SRC), )
+        $(foreach SOURCE, $(EXT_SRC), $(eval SRC += $(KEYMAP_PATH)/$(SOURCE)))
+    endif
 endif
 
 # Generate the keymap.c

build_keyboard.mk

@@ -100,6 +100,35 @@ MAIN_KEYMAP_PATH_5 := $(KEYBOARD_PATH_5)/keymaps/$(KEYMAP)
 # Check for keymap.json first, so we can regenerate keymap.c
 include build_json.mk
 
+
+ifneq ($(EXTERNAL_USERSPACE), )
+    # Look for out-of-tree keyboard-specific keymap
+    ifneq ("$(wildcard $(EXTERNAL_USERSPACE)/keyboards/$(KEYBOARD_FOLDER_PATH_5)/$(KEYMAP)/keymap.c)", "")
+        KEYMAP_PATH := $(EXTERNAL_USERSPACE)/keyboards/$(KEYBOARD_FOLDER_PATH_5)/$(KEYMAP)
+        KEYMAP_C := $(EXTERNAL_USERSPACE)/keyboards/$(KEYBOARD_FOLDER_PATH_5)/$(KEYMAP)/keymap.c
+    else ifneq ("$(wildcard $(EXTERNAL_USERSPACE)/keyboards/$(KEYBOARD_FOLDER_PATH_4)/$(KEYMAP)/keymap.c)", "")
+        KEYMAP_PATH := $(EXTERNAL_USERSPACE)/keyboards/$(KEYBOARD_FOLDER_PATH_4)/$(KEYMAP)
+        KEYMAP_C := $(EXTERNAL_USERSPACE)/keyboards/$(KEYBOARD_FOLDER_PATH_4)/$(KEYMAP)/keymap.c
+    else ifneq ("$(wildcard $(EXTERNAL_USERSPACE)/keyboards/$(KEYBOARD_FOLDER_PATH_3)/$(KEYMAP)/keymap.c)", "")
+        KEYMAP_PATH := $(EXTERNAL_USERSPACE)/keyboards/$(KEYBOARD_FOLDER_PATH_3)/$(KEYMAP)
+        KEYMAP_C := $(EXTERNAL_USERSPACE)/keyboards/$(KEYBOARD_FOLDER_PATH_3)/$(KEYMAP)/keymap.c
+    else ifneq ("$(wildcard $(EXTERNAL_USERSPACE)/keyboards/$(KEYBOARD_FOLDER_PATH_2)/$(KEYMAP)/keymap.c)", "")
+        KEYMAP_PATH := $(EXTERNAL_USERSPACE)/keyboards/$(KEYBOARD_FOLDER_PATH_2)/$(KEYMAP)
+        KEYMAP_C := $(EXTERNAL_USERSPACE)/keyboards/$(KEYBOARD_FOLDER_PATH_2)/$(KEYMAP)/keymap.c
+    else ifneq ("$(wildcard $(EXTERNAL_USERSPACE)/keyboards/$(KEYBOARD_FOLDER_PATH_1)/$(KEYMAP)/keymap.c)", "")
+        KEYMAP_PATH := $(EXTERNAL_USERSPACE)/keyboards/$(KEYBOARD_FOLDER_PATH_1)/$(KEYMAP)
+        KEYMAP_C := $(EXTERNAL_USERSPACE)/keyboards/$(KEYBOARD_FOLDER_PATH_1)/$(KEYMAP)/keymap.c
+    endif
+
+    ifneq ("$(wildcard $(KEYMAP_PATH))", "")
+        -include $(KEYMAP_PATH)/rules.mk
+        # If EXT_SRC exists, add all files to SRC with the EXT_KM_PATH prefix so make can find it
+        ifneq ($(EXT_SRC), )
+            $(foreach SOURCE, $(EXT_SRC), $(eval SRC += $(KEYMAP_PATH)/$(SOURCE)))
+        endif
+    endif
+endif
+
 ifeq ("$(wildcard $(KEYMAP_PATH))", "")
     # Look through the possible keymap folders until we find a matching keymap.c
     ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_5)/keymap.c)","")
@@ -283,6 +312,19 @@ ifneq ("$(wildcard $(USER_PATH)/config.h)","")
     CONFIG_H += $(USER_PATH)/config.h
 endif
 
+ifneq ($(EXTERNAL_USERSPACE), )
+    # Look for out-of-tree userspace stuff, if set up
+    COMMON_PATH := $(EXTERNAL_USERSPACE)/common
+    -include $(COMMON_PATH)/rules.mk
+    # If EXT_SRC exists, add all files to SRC with the EXT_KM_PATH prefix so make can find it
+    ifneq ($(EXT_SRC), )
+        $(foreach SOURCE, $(EXT_SRC), $(eval SRC += $(COMMON_PATH)/$(SOURCE))) 
+    endif
+    ifneq ("$(wildcard $(COMMON_PATH)/config.h)","")
+        CONFIG_H += $(COMMON_PATH)/config.h
+    endif
+endif
+
 # Object files directory
 #     To put object files in current directory, use a dot (.), do NOT make
 #     this an empty or blank macro!

build_layout.mk

@@ -1,5 +1,11 @@
+# Load the out-of-tree layout-specific keymap, if requested
+ifneq ($(EXTERNAL_USERSPACE),)
+    LAYOUTS_PATH := $(EXTERNAL_USERSPACE)/layouts
+    LAYOUTS_REPOS := $(patsubst %/,%,$(sort $(dir $(wildcard $(LAYOUTS_PATH)/))))
+endif
+
 LAYOUTS_PATH := layouts
-LAYOUTS_REPOS := $(patsubst %/,%,$(sort $(dir $(wildcard $(LAYOUTS_PATH)/*/))))
+LAYOUTS_REPOS += $(patsubst %/,%,$(sort $(dir $(wildcard $(LAYOUTS_PATH)/*/))))
 
 define SEARCH_LAYOUTS_REPO
     LAYOUT_KEYMAP_PATH := $$(LAYOUTS_REPO)/$$(LAYOUT)/$$(KEYMAP)

docs/_summary.md

@@ -87,6 +87,7 @@
     * [Terminal](feature_terminal.md)
     * [Unicode](feature_unicode.md)
     * [Userspace](feature_userspace.md)
+    * [External Userspace](feature_external_userspace.md)
     * [WPM Calculation](feature_wpm.md)
 
   * Hardware Features

docs/config_options.md

@@ -312,6 +312,8 @@ This is a [make](https://www.gnu.org/software/make/manual/make.html) file that i
   * Defines which format (bin, hex) is copied to the root `qmk_firmware` folder after building.
 * `SRC`
   * Used to add files to the compilation/linking list.
+* `EXT_SRC`
+  * Used to add files to the compilation/linking list. (used with [External Userspace](feature_external_userspace.md))
 * `LIB_SRC`
   * Used to add files as a library to the compilation/linking list.  
     The files specified by `LIB_SRC` is linked after the files specified by `SRC`.  

docs/feature_external_userspace.md

@@ -0,0 +1,69 @@
+# External Userspace: Store your code outside the QMK repo
+
+Store your [keymaps](keymap.md),  [community layouts](feature_layouts.md) and [userspace](feature_userspace.md) outside the `qmk_firmware` repository and automatically link it during build time.
+This allows you to store your code in a separate `git` repository and manage it separately.
+Updating the QMK repo should be easier as well, as the chance of code conflict is basically non-existent.
+
+## Directory structure
+*Note*: all directories and files are optional, create only what you actually need
+
+* `/my_qmk_stuff/` (put it anywhere you'd like and name is whatever you want (no whitespaces, please))
+  * `common/` (content will be always included, equivalent of [userspace](feature_userspace.md)))
+    * `rules.mk`
+    * `config.h`
+    * `<name>.h`
+    * `<name>.c`
+    * `cool_rgb_stuff.c`
+    * `cool_rgb_stuff.h`
+  * `keyboards/` (keyboard-specific [keymaps](keymap.md))
+    * `planck/`
+      * `my_planck_keymap/` (keymaps for all Planck revisions)
+        * `keymap.c`
+        * `config.h`
+      * `rev6` (keymaps only for the `planck/rev6`)
+        * `my_rev6_keymap/`
+          * `keymap.json`
+          * `rules.mk` 
+  * `layouts/` (layout specific keymaps, equivalent of [community layouts](feature_layouts.md))
+    * `ortho_4x12/`
+      * `my_grid_keymap/`
+        * `keymap.c`
+
+## Configuration
+
+    qmk config user.userspace=/path/to/external_userspace
+
+**Note**: It's important to specify absolute path.
+
+*Example*: I created my external userspace in `/home/erovia/qmk/my_qmk_stuff`, so I have to run this command:
+
+    qmk config user.userspace=/home/erovia/qmk/my_qmk/stuff
+
+`qmk doctor` will check if the directory exists and if the provided path is an absolute one.
+
+## Additional source files
+
+You can add additional source files to the compilation with the EXT_SRC option, which is very similar to [SRC](config_options?id=build-options) used in in-tree keymaps.
+You need to add the EXT_SRC in `rules.mk` like this:
+
+    EXT_SRC += <filename>.c
+
+Additional files may be added in the same way.
+
+Similarly to [userspace](feature_userspace.md), the `common/rules.mk` file will be included in the build _after_ the `rules.mk` from your keymap. This allows you to have features in your userspace `rules.mk` that depend on individual QMK features that may or may not be available on a specific keyboard. 
+
+For example, if you have RGB control features shared between all your keyboards that support RGB lighting, you can add support for that if the RGBLIGHT feature is enabled:
+```make
+ifeq ($(strip $(RGBLIGHT_ENABLE)), yes)
+  # Include my fancy rgb functions source here
+  EXT_SRC += cool_rgb_stuff.c
+endif
+```
+
+Alternatively, you can `define RGB_ENABLE` in your keymap's `rules.mk` and then check for the variable in your userspace's `rules.mk` like this:
+```make
+ifdef RGB_ENABLE
+  # Include my fancy rgb functions source here
+  EXT_SRC += cool_rgb_stuff.c
+endif
+```

lib/python/qmk/cli/doctor.py

@@ -5,11 +5,8 @@
 import platform
 
 from milc import cli
-from milc.questions import yesno
-from qmk import submodules
 from qmk.constants import QMK_FIRMWARE
-from qmk.commands import run
-from qmk.os_helpers import CheckStatus, check_binaries, check_binary_versions, check_submodules, check_git_repo
+from qmk.os_helpers import CheckStatus, check_userspace, repo_test, tooling_test
 
 
 def os_tests():
@@ -66,50 +63,19 @@ def doctor(cli):
     """
     cli.log.info('QMK Doctor is checking your environment.')
 
-    status = os_tests()
-
     cli.log.info('QMK home: {fg_cyan}%s', QMK_FIRMWARE)
 
-    # Make sure our QMK home is a Git repo
-    git_ok = check_git_repo()
-
-    if git_ok == CheckStatus.WARNING:
-        cli.log.warning("QMK home does not appear to be a Git repository! (no .git folder)")
-        status = CheckStatus.WARNING
-
-    # Make sure the basic CLI tools we need are available and can be executed.
-    bin_ok = check_binaries()
+    status = CheckStatus.OK
 
-    if not bin_ok:
-        if yesno('Would you like to install dependencies?', default=True):
-            run(['util/qmk_install.sh'])
-            bin_ok = check_binaries()
-
-    if bin_ok:
-        cli.log.info('All dependencies are installed.')
-    else:
-        status = CheckStatus.ERROR
-
-    # Make sure the tools are at the correct version
-    ver_ok = check_binary_versions()
-    if CheckStatus.ERROR in ver_ok:
-        status = CheckStatus.ERROR
-    elif CheckStatus.WARNING in ver_ok and status == CheckStatus.OK:
-        status = CheckStatus.WARNING
-
-    # Check out the QMK submodules
-    sub_ok = check_submodules()
-
-    if sub_ok == CheckStatus.OK:
-        cli.log.info('Submodules are up to date.')
-    else:
-        if yesno('Would you like to clone the submodules?', default=True):
-            submodules.update()
-            sub_ok = check_submodules()
+    # List of checks we need to run
+    checks = (os_tests, check_userspace, repo_test, tooling_test)
 
-        if CheckStatus.ERROR in sub_ok:
+    # Run the checks and update 'status' based on their output
+    for check in checks:
+        check_status = check()
+        if check_status == CheckStatus.ERROR:
             status = CheckStatus.ERROR
-        elif CheckStatus.WARNING in sub_ok and status == CheckStatus.OK:
+        elif check_status == CheckStatus.WARNING:
             status = CheckStatus.WARNING
 
     # Report a summary of our findings to the user

lib/python/qmk/commands.py

@@ -7,6 +7,8 @@
 import shlex
 import shutil
 
+from milc import cli
+
 import qmk.keymap
 
 
@@ -34,7 +36,12 @@ def create_make_command(keyboard, keymap, target=None):
     if target:
         make_args.append(target)
 
-    return [make_cmd, ':'.join(make_args)]
+    make_command = [make_cmd, ':'.join(make_args)]
+
+    if cli.config.user.userspace:
+        make_command.append('EXTERNAL_USERSPACE=%s' % cli.config.user.userspace)
+
+    return make_command
 
 
 def compile_configurator_json(user_keymap, bootloader=None):