Fix keycode parameter extraction to match the new DD keycodes (#18977)

* Add macros to extract parameters from keycode values

Implement both encoding and decoding for keycodes like TO(layer) or
LM(layer, mod) in one place, so that the decoding won't get out of sync
with the encoding.

While at it, fix some macros for creating keycode values that did not
apply the appropriate masks to parameters (and therefore could allow the
result to be out of range if a wrong parameter was passed).

* keymap_common: Use extraction macros for keycodes

* pointing_device_auto_mouse: Use extraction macros for keycodes

Fixes #18970.

* process_autocorrect: Use extraction macros for keycodes

* process_caps_word: Use extraction macros for keycodes

(Also fix a minor bug - SH_TG was not handled properly)

* process_leader: Use extraction macros for keycodes

(Technically the code is not 100% correct, because it always assumes
that the LT() or MT() action was a tap, but it's a separate issue that
already existed before the keycode changes.)

* process_unicode: Use extraction macros for keycodes

* process_unicodemap: Use extraction macros for keycodes

8 files changed, 81 insertions, 38 deletions

diff --git a/quantum/keymap_common.c b/quantum/keymap_common.c
index a5997711f2..c4336440f9 100644
--- a/quantum/keymap_common.c
+++ b/quantum/keymap_common.c
@@ -74,64 +74,64 @@ action_t action_for_keycode(uint16_t keycode) {
         case QK_MODS ... QK_MODS_MAX:;
             // Has a modifier
             // Split it up
-            action.code = ACTION_MODS_KEY(keycode >> 8, keycode & 0xFF); // adds modifier to key
+            action.code = ACTION_MODS_KEY(QK_MODS_GET_MODS(keycode), QK_MODS_GET_BASIC_KEYCODE(keycode)); // adds modifier to key
             break;
 #ifndef NO_ACTION_LAYER
         case QK_LAYER_TAP ... QK_LAYER_TAP_MAX:
-            action.code = ACTION_LAYER_TAP_KEY((keycode >> 0x8) & 0xF, keycode & 0xFF);
+            action.code = ACTION_LAYER_TAP_KEY(QK_LAYER_TAP_GET_LAYER(keycode), QK_LAYER_TAP_GET_TAP_KEYCODE(keycode));
             break;
         case QK_TO ... QK_TO_MAX:;
             // Layer set "GOTO"
-            action_layer = keycode & 0xFF;
+            action_layer = QK_TO_GET_LAYER(keycode);
             action.code  = ACTION_LAYER_GOTO(action_layer);
             break;
         case QK_MOMENTARY ... QK_MOMENTARY_MAX:;
             // Momentary action_layer
-            action_layer = keycode & 0xFF;
+            action_layer = QK_MOMENTARY_GET_LAYER(keycode);
             action.code  = ACTION_LAYER_MOMENTARY(action_layer);
             break;
         case QK_DEF_LAYER ... QK_DEF_LAYER_MAX:;
             // Set default action_layer
-            action_layer = keycode & 0xFF;
+            action_layer = QK_DEF_LAYER_GET_LAYER(keycode);
             action.code  = ACTION_DEFAULT_LAYER_SET(action_layer);
             break;
         case QK_TOGGLE_LAYER ... QK_TOGGLE_LAYER_MAX:;
             // Set toggle
-            action_layer = keycode & 0xFF;
+            action_layer = QK_TOGGLE_LAYER_GET_LAYER(keycode);
             action.code  = ACTION_LAYER_TOGGLE(action_layer);
             break;
 #endif
 #ifndef NO_ACTION_ONESHOT
         case QK_ONE_SHOT_LAYER ... QK_ONE_SHOT_LAYER_MAX:;
             // OSL(action_layer) - One-shot action_layer
-            action_layer = keycode & 0xFF;
+            action_layer = QK_ONE_SHOT_LAYER_GET_LAYER(keycode);
             action.code  = ACTION_LAYER_ONESHOT(action_layer);
             break;
         case QK_ONE_SHOT_MOD ... QK_ONE_SHOT_MOD_MAX:;
             // OSM(mod) - One-shot mod
-            mod         = mod_config(keycode & 0x1F);
+            mod         = mod_config(QK_ONE_SHOT_MOD_GET_MODS(keycode));
             action.code = ACTION_MODS_ONESHOT(mod);
             break;
 #endif
 #ifndef NO_ACTION_LAYER
         case QK_LAYER_TAP_TOGGLE ... QK_LAYER_TAP_TOGGLE_MAX:
-            action.code = ACTION_LAYER_TAP_TOGGLE(keycode & 0xFF);
+            action.code = ACTION_LAYER_TAP_TOGGLE(QK_LAYER_TAP_TOGGLE_GET_LAYER(keycode));
             break;
         case QK_LAYER_MOD ... QK_LAYER_MOD_MAX:
-            mod          = mod_config(keycode & 0x1F);
-            action_layer = (keycode >> 5) & 0xF;
+            mod          = mod_config(QK_LAYER_MOD_GET_MODS(keycode));
+            action_layer = QK_LAYER_MOD_GET_LAYER(keycode);
             action.code  = ACTION_LAYER_MODS(action_layer, mod);
             break;
 #endif
 #ifndef NO_ACTION_TAPPING
         case QK_MOD_TAP ... QK_MOD_TAP_MAX:
-            mod         = mod_config((keycode >> 0x8) & 0x1F);
-            action.code = ACTION_MODS_TAP_KEY(mod, keycode & 0xFF);
+            mod         = mod_config(QK_MOD_TAP_GET_MODS(keycode));
+            action.code = ACTION_MODS_TAP_KEY(mod, QK_MOD_TAP_GET_TAP_KEYCODE(keycode));
             break;
 #endif
 #ifdef SWAP_HANDS_ENABLE
         case QK_SWAP_HANDS ... QK_SWAP_HANDS_MAX:
-            action.code = ACTION(ACT_SWAP_HANDS, keycode & 0xff);
+            action.code = ACTION(ACT_SWAP_HANDS, QK_SWAP_HANDS_GET_TAP_KEYCODE(keycode));
             break;
 #endif
 
diff --git a/quantum/pointing_device/pointing_device_auto_mouse.c b/quantum/pointing_device/pointing_device_auto_mouse.c
index edffd44787..5e78817c7c 100644
--- a/quantum/pointing_device/pointing_device_auto_mouse.c
+++ b/quantum/pointing_device/pointing_device_auto_mouse.c
@@ -266,16 +266,20 @@ bool process_auto_mouse(uint16_t keycode, keyrecord_t* record) {
         case QK_MODS ... QK_MODS_MAX:
             break;
         // TO((AUTO_MOUSE_TARGET_LAYER))-------------------------------------------------------------------------------
-        case QK_TO ... QK_TO_MAX: // same proccessing as next
+        case QK_TO ... QK_TO_MAX:
+            if (QK_TO_GET_LAYER(keycode) == (AUTO_MOUSE_TARGET_LAYER)) {
+                if (!(record->event.pressed)) auto_mouse_toggle();
+            }
+            break;
         // TG((AUTO_MOUSE_TARGET_LAYER))-------------------------------------------------------------------------------
         case QK_TOGGLE_LAYER ... QK_TOGGLE_LAYER_MAX:
-            if ((keycode & 0xff) == (AUTO_MOUSE_TARGET_LAYER)) {
+            if (QK_TOGGLE_LAYER_GET_LAYER(keycode) == (AUTO_MOUSE_TARGET_LAYER)) {
                 if (!(record->event.pressed)) auto_mouse_toggle();
             }
             break;
         // MO((AUTO_MOUSE_TARGET_LAYER))-------------------------------------------------------------------------------
         case QK_MOMENTARY ... QK_MOMENTARY_MAX:
-            if ((keycode & 0xff) == (AUTO_MOUSE_TARGET_LAYER)) {
+            if (QK_MOMENTARY_GET_LAYER(keycode) == (AUTO_MOUSE_TARGET_LAYER)) {
                 auto_mouse_keyevent(record->event.pressed);
             }
         // DF ---------------------------------------------------------------------------------------------------------
@@ -288,14 +292,14 @@ bool process_auto_mouse(uint16_t keycode, keyrecord_t* record) {
             break;
         // LM((AUTO_MOUSE_TARGET_LAYER), mod)--------------------------------------------------------------------------
         case QK_LAYER_MOD ... QK_LAYER_MOD_MAX:
-            if (((keycode >> 8) & 0x0f) == (AUTO_MOUSE_TARGET_LAYER)) {
+            if (QK_LAYER_MOD_GET_LAYER(keycode) == (AUTO_MOUSE_TARGET_LAYER)) {
                 auto_mouse_keyevent(record->event.pressed);
             }
             break;
             // TT((AUTO_MOUSE_TARGET_LAYER))---------------------------------------------------------------------------
 #    ifndef NO_ACTION_TAPPING
         case QK_LAYER_TAP_TOGGLE ... QK_LAYER_TAP_TOGGLE_MAX:
-            if ((keycode & 0xff) == (AUTO_MOUSE_TARGET_LAYER)) {
+            if (QK_LAYER_TAP_TOGGLE_GET_LAYER(keycode) == (AUTO_MOUSE_TARGET_LAYER)) {
                 auto_mouse_keyevent(record->event.pressed);
 #        if TAPPING_TOGGLE != 0
                 if (record->tap.count == TAPPING_TOGGLE) {
@@ -312,7 +316,7 @@ bool process_auto_mouse(uint16_t keycode, keyrecord_t* record) {
         // LT((AUTO_MOUSE_TARGET_LAYER), kc)---------------------------------------------------------------------------
         case QK_LAYER_TAP ... QK_LAYER_TAP_MAX:
             if (!record->tap.count) {
-                if (((keycode >> 8) & 0x0f) == (AUTO_MOUSE_TARGET_LAYER)) {
+                if (QK_LAYER_TAP_GET_LAYER(keycode) == (AUTO_MOUSE_TARGET_LAYER)) {
                     auto_mouse_keyevent(record->event.pressed);
                 }
                 break;
diff --git a/quantum/process_keycode/process_autocorrect.c b/quantum/process_keycode/process_autocorrect.c
index e3fcbc34ae..8aeebf0e06 100644
--- a/quantum/process_keycode/process_autocorrect.c
+++ b/quantum/process_keycode/process_autocorrect.c
@@ -91,7 +91,7 @@ __attribute__((weak)) bool process_autocorrect_user(uint16_t *keycode, keyrecord
             } else {
                 *mods |= MOD_RSFT;
             }
-            *keycode &= 0xFF; // Get the basic keycode.
+            *keycode = QK_MODS_GET_BASIC_KEYCODE(*keycode); // Get the basic keycode.
             return true;
 #ifndef NO_ACTION_TAPPING
         // Exclude tap-hold keys when they are held down
@@ -101,13 +101,20 @@ __attribute__((weak)) bool process_autocorrect_user(uint16_t *keycode, keyrecord
             // Exclude Layer Tap, if layers are disabled
             // but action tapping is still enabled.
             return false;
+#    else
+            // Exclude hold keycode
+            if (!record->tap.count) {
+                return false;
+            }
+            *keycode = QK_LAYER_TAP_GET_TAP_KEYCODE(*keycode);
+            break;
 #    endif
         case QK_MOD_TAP ... QK_MOD_TAP_MAX:
             // Exclude hold keycode
             if (!record->tap.count) {
                 return false;
             }
-            *keycode &= 0xFF;
+            *keycode = QK_MOD_TAP_GET_TAP_KEYCODE(*keycode);
             break;
 #else
         case QK_MOD_TAP ... QK_MOD_TAP_MAX:
@@ -119,10 +126,12 @@ __attribute__((weak)) bool process_autocorrect_user(uint16_t *keycode, keyrecord
         // and mask for base keycode when they are tapped.
         case QK_SWAP_HANDS ... QK_SWAP_HANDS_MAX:
 #ifdef SWAP_HANDS_ENABLE
-            if (*keycode >= 0x56F0 || !record->tap.count) {
+            // Note: IS_SWAP_HANDS_KEYCODE() actually tests for the special action keycodes like SH_TG, SH_TT, ...,
+            // which currently overlap the SH_T(kc) range.
+            if (IS_SWAP_HANDS_KEYCODE(*keycode) || !record->tap.count) {
                 return false;
             }
-            *keycode &= 0xFF;
+            *keycode = QK_SWAP_HANDS_GET_TAP_KEYCODE(*keycode);
             break;
 #else
             // Exclude if disabled
diff --git a/quantum/process_keycode/process_caps_word.c b/quantum/process_keycode/process_caps_word.c
index 7a83f7bf49..4c0217eba7 100644
--- a/quantum/process_keycode/process_caps_word.c
+++ b/quantum/process_keycode/process_caps_word.c
@@ -109,7 +109,7 @@ bool process_caps_word(uint16_t keycode, keyrecord_t* record) {
             // * Otherwise stop Caps Word.
             case QK_MOD_TAP ... QK_MOD_TAP_MAX:
                 if (record->tap.count == 0) { // Mod-tap key is held.
-                    const uint8_t mods = (keycode >> 8) & 0x1f;
+                    const uint8_t mods = QK_MOD_TAP_GET_MODS(keycode);
                     switch (mods) {
                         case MOD_LSFT:
                             keycode = KC_LSFT;
@@ -127,7 +127,7 @@ bool process_caps_word(uint16_t keycode, keyrecord_t* record) {
                             return true;
                     }
                 } else {
-                    keycode &= 0xff;
+                    keycode = QK_MOD_TAP_GET_TAP_KEYCODE(keycode);
                 }
                 break;
 
@@ -137,16 +137,18 @@ bool process_caps_word(uint16_t keycode, keyrecord_t* record) {
                 if (record->tap.count == 0) {
                     return true;
                 }
-                keycode &= 0xff;
+                keycode = QK_LAYER_TAP_GET_TAP_KEYCODE(keycode);
                 break;
 #endif // NO_ACTION_TAPPING
 
 #ifdef SWAP_HANDS_ENABLE
             case QK_SWAP_HANDS ... QK_SWAP_HANDS_MAX:
-                if (keycode > 0x56F0 || record->tap.count == 0) {
+                // Note: IS_SWAP_HANDS_KEYCODE() actually tests for the special action keycodes like SH_TG, SH_TT, ...,
+                // which currently overlap the SH_T(kc) range.
+                if (IS_SWAP_HANDS_KEYCODE(keycode) || record->tap.count == 0) {
                     return true;
                 }
-                keycode &= 0xff;
+                keycode = QK_SWAP_HANDS_GET_TAP_KEYCODE(keycode);
                 break;
 #endif // SWAP_HANDS_ENABLE
         }
diff --git a/quantum/process_keycode/process_leader.c b/quantum/process_keycode/process_leader.c
index 7e6f3ad73c..b74b4927a8 100644
--- a/quantum/process_keycode/process_leader.c
+++ b/quantum/process_keycode/process_leader.c
@@ -54,8 +54,10 @@ bool process_leader(uint16_t keycode, keyrecord_t *record) {
 #    endif // LEADER_NO_TIMEOUT
             {
 #    ifndef LEADER_KEY_STRICT_KEY_PROCESSING
-                if ((keycode >= QK_MOD_TAP && keycode <= QK_MOD_TAP_MAX) || (keycode >= QK_LAYER_TAP && keycode <= QK_LAYER_TAP_MAX)) {
-                    keycode = keycode & 0xFF;
+                if (IS_QK_MOD_TAP(keycode)) {
+                    keycode = QK_MOD_TAP_GET_TAP_KEYCODE(keycode);
+                } else if (IS_QK_LAYER_TAP(keycode)) {
+                    keycode = QK_LAYER_TAP_GET_TAP_KEYCODE(keycode);
                 }
 #    endif // LEADER_KEY_STRICT_KEY_PROCESSING
                 if (leader_sequence_size < ARRAY_SIZE(leader_sequence)) {
diff --git a/quantum/process_keycode/process_unicode.c b/quantum/process_keycode/process_unicode.c
index 99cc2f5f26..1ec76245a3 100644
--- a/quantum/process_keycode/process_unicode.c
+++ b/quantum/process_keycode/process_unicode.c
@@ -21,7 +21,7 @@
 bool process_unicode(uint16_t keycode, keyrecord_t *record) {
     if (record->event.pressed) {
         if (keycode >= QK_UNICODE && keycode <= QK_UNICODE_MAX) {
-            register_unicode(keycode & 0x7FFF);
+            register_unicode(QK_UNICODE_GET_CODE_POINT(keycode));
         }
     }
     return true;
diff --git a/quantum/process_keycode/process_unicodemap.c b/quantum/process_keycode/process_unicodemap.c
index 979d773b05..195c093e6e 100644
--- a/quantum/process_keycode/process_unicodemap.c
+++ b/quantum/process_keycode/process_unicodemap.c
@@ -24,7 +24,7 @@
 __attribute__((weak)) uint16_t unicodemap_index(uint16_t keycode) {
     if (keycode >= QK_UNICODEMAP_PAIR) {
         // Keycode is a pair: extract index based on Shift / Caps Lock state
-        uint16_t index = keycode - QK_UNICODEMAP_PAIR;
+        uint16_t index;
 
         uint8_t mods = get_mods() | get_weak_mods();
 #ifndef NO_ACTION_ONESHOT
@@ -34,13 +34,15 @@ __attribute__((weak)) uint16_t unicodemap_index(uint16_t keycode) {
         bool shift = mods & MOD_MASK_SHIFT;
         bool caps  = host_keyboard_led_state().caps_lock;
         if (shift ^ caps) {
-            index >>= 7;
+            index = QK_UNICODEMAP_PAIR_GET_SHIFTED_INDEX(keycode);
+        } else {
+            index = QK_UNICODEMAP_PAIR_GET_UNSHIFTED_INDEX(keycode);
         }
 
-        return index & 0x7F;
+        return index;
     } else {
         // Keycode is a regular index
-        return keycode - QK_UNICODEMAP;
+        return QK_UNICODEMAP_GET_INDEX(keycode);
     }
 }
 
diff --git a/quantum/quantum_keycodes.h b/quantum/quantum_keycodes.h
index ec69fadbab..6b0e890792 100644
--- a/quantum/quantum_keycodes.h
+++ b/quantum/quantum_keycodes.h
@@ -39,6 +39,10 @@
 #define QK_UNICODEMAP_PAIR_MAX 0xFFFF
 // clang-format on
 
+// Generic decoding for the whole QK_MODS range
+#define QK_MODS_GET_MODS(kc) (((kc) >> 8) & 0x1F)
+#define QK_MODS_GET_BASIC_KEYCODE(kc) ((kc)&0xFF)
+
 // Keycode modifiers & aliases
 #define LCTL(kc) (QK_LCTL | (kc))
 #define LSFT(kc) (QK_LSFT | (kc))
@@ -80,33 +84,46 @@
 
 // GOTO layer - 32 layer max
 #define TO(layer) (QK_TO | ((layer)&0x1F))
+#define QK_TO_GET_LAYER(kc) ((kc)&0x1F)
 
 // Momentary switch layer - 32 layer max
 #define MO(layer) (QK_MOMENTARY | ((layer)&0x1F))
+#define QK_MOMENTARY_GET_LAYER(kc) ((kc)&0x1F)
 
 // Set default layer - 32 layer max
 #define DF(layer) (QK_DEF_LAYER | ((layer)&0x1F))
+#define QK_DEF_LAYER_GET_LAYER(kc) ((kc)&0x1F)
 
 // Toggle to layer - 32 layer max
 #define TG(layer) (QK_TOGGLE_LAYER | ((layer)&0x1F))
+#define QK_TOGGLE_LAYER_GET_LAYER(kc) ((kc)&0x1F)
 
 // One-shot layer - 32 layer max
 #define OSL(layer) (QK_ONE_SHOT_LAYER | ((layer)&0x1F))
+#define QK_ONE_SHOT_LAYER_GET_LAYER(kc) ((kc)&0x1F)
 
 // L-ayer M-od: Momentary switch layer with modifiers active - 16 layer max
 #define LM(layer, mod) (QK_LAYER_MOD | (((layer)&0xF) << 5) | ((mod)&0x1F))
+#define QK_LAYER_MOD_GET_LAYER(kc) (((kc) >> 5) & 0xF)
+#define QK_LAYER_MOD_GET_MODS(kc) ((kc)&0x1F)
 
 // One-shot mod
 #define OSM(mod) (QK_ONE_SHOT_MOD | ((mod)&0x1F))
+#define QK_ONE_SHOT_MOD_GET_MODS(kc) ((kc)&0x1F)
 
 // Layer tap-toggle - 32 layer max
 #define TT(layer) (QK_LAYER_TAP_TOGGLE | ((layer)&0x1F))
+#define QK_LAYER_TAP_TOGGLE_GET_LAYER(kc) ((kc)&0x1F)
 
 // L-ayer, T-ap - 256 keycode max, 16 layer max
 #define LT(layer, kc) (QK_LAYER_TAP | (((layer)&0xF) << 8) | ((kc)&0xFF))
+#define QK_LAYER_TAP_GET_LAYER(kc) (((kc) >> 8) & 0xF)
+#define QK_LAYER_TAP_GET_TAP_KEYCODE(kc) ((kc)&0xFF)
 
 // M-od, T-ap - 256 keycode max
 #define MT(mod, kc) (QK_MOD_TAP | (((mod)&0x1F) << 8) | ((kc)&0xFF))
+#define QK_MOD_TAP_GET_MODS(kc) (((kc) >> 8) & 0x1F)
+#define QK_MOD_TAP_GET_TAP_KEYCODE(kc) ((kc)&0xFF)
 
 #define LCTL_T(kc) MT(MOD_LCTL, kc)
 #define RCTL_T(kc) MT(MOD_RCTL, kc)
@@ -161,12 +178,19 @@
 // Unicode aliases
 // UNICODE_ENABLE - Allows Unicode input up to 0x7FFF
 #define UC(c) (QK_UNICODE | (c))
+#define QK_UNICODE_GET_CODE_POINT(kc) ((kc)&0x7FFF)
+
 // UNICODEMAP_ENABLE - Allows Unicode input up to 0x10FFFF, requires unicode_map
-#define X(i) (QK_UNICODEMAP | (i))
+#define X(i) (QK_UNICODEMAP | ((i)&0x3FFF))
+#define QK_UNICODEMAP_GET_INDEX(kc) ((kc)&0x3FFF)
+
 #define XP(i, j) (QK_UNICODEMAP_PAIR | ((i)&0x7F) | (((j)&0x7F) << 7)) // 127 max i and j
+#define QK_UNICODEMAP_PAIR_GET_UNSHIFTED_INDEX(kc) ((kc)&0x7F)
+#define QK_UNICODEMAP_PAIR_GET_SHIFTED_INDEX(kc) (((kc) >> 7) & 0x7F)
 
 // Swap Hands
-#define SH_T(kc) (QK_SWAP_HANDS | (kc))
+#define SH_T(kc) (QK_SWAP_HANDS | ((kc)&0xFF))
+#define QK_SWAP_HANDS_GET_TAP_KEYCODE(kc) ((kc)&0xFF)
 
 // MIDI aliases
 #define MIDI_TONE_MIN MI_C