Improve layer mask handling (#13065)

2 files changed, 33 insertions, 8 deletions

diff --git a/tmk_core/common/action_layer.c b/tmk_core/common/action_layer.c
index af2d7d964b..ed1a4bd20d 100644
--- a/tmk_core/common/action_layer.c
+++ b/tmk_core/common/action_layer.c
@@ -131,32 +131,32 @@ bool layer_state_cmp(layer_state_t cmp_layer_state, uint8_t layer) {
     if (!cmp_layer_state) {
         return layer == 0;
     }
-    return (cmp_layer_state & (1UL << layer)) != 0;
+    return (cmp_layer_state & ((layer_state_t)1 << layer)) != 0;
 }
 
 /** \brief Layer move
  *
  * Turns on the given layer and turn off all other layers
  */
-void layer_move(uint8_t layer) { layer_state_set(1UL << layer); }
+void layer_move(uint8_t layer) { layer_state_set((layer_state_t)1 << layer); }
 
 /** \brief Layer on
  *
  * Turns on given layer
  */
-void layer_on(uint8_t layer) { layer_state_set(layer_state | (1UL << layer)); }
+void layer_on(uint8_t layer) { layer_state_set(layer_state | ((layer_state_t)1 << layer)); }
 
 /** \brief Layer off
  *
  * Turns off given layer
  */
-void layer_off(uint8_t layer) { layer_state_set(layer_state & ~(1UL << layer)); }
+void layer_off(uint8_t layer) { layer_state_set(layer_state & ~((layer_state_t)1 << layer)); }
 
 /** \brief Layer invert
  *
  * Toggle the given layer (set it if it's unset, or unset it if it's set)
  */
-void layer_invert(uint8_t layer) { layer_state_set(layer_state ^ (1UL << layer)); }
+void layer_invert(uint8_t layer) { layer_state_set(layer_state ^ ((layer_state_t)1 << layer)); }
 
 /** \brief Layer or
  *
@@ -258,7 +258,7 @@ uint8_t layer_switch_get_layer(keypos_t key) {
     layer_state_t layers = layer_state | default_layer_state;
     /* check top layer first */
     for (int8_t i = MAX_LAYER - 1; i >= 0; i--) {
-        if (layers & (1UL << i)) {
+        if (layers & ((layer_state_t)1 << i)) {
             action = action_for_key(i, key);
             if (action.code != ACTION_TRANSPARENT) {
                 return i;
diff --git a/tmk_core/common/action_layer.h b/tmk_core/common/action_layer.h
index d72cd3e3a5..b87d096eed 100644
--- a/tmk_core/common/action_layer.h
+++ b/tmk_core/common/action_layer.h
@@ -21,6 +21,29 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #include "keyboard.h"
 #include "action.h"
 
+#ifdef DYNAMIC_KEYMAP_ENABLE
+#    ifndef DYNAMIC_KEYMAP_LAYER_COUNT
+#        define DYNAMIC_KEYMAP_LAYER_COUNT 4
+#    endif
+#    if DYNAMIC_KEYMAP_LAYER_COUNT <= 8
+#        ifndef LAYER_STATE_8BIT
+#            define LAYER_STATE_8BIT
+#        endif
+#    elif DYNAMIC_KEYMAP_LAYER_COUNT <= 16
+#        ifndef LAYER_STATE_16BIT
+#            define LAYER_STATE_16BIT
+#        endif
+#    else
+#        ifndef LAYER_STATE_32BIT
+#            define LAYER_STATE_32BIT
+#        endif
+#    endif
+#endif
+
+#if !defined(LAYER_STATE_8BIT) && !defined(LAYER_STATE_16BIT) && !defined(LAYER_STATE_32BIT)
+#    define LAYER_STATE_32BIT
+#endif
+
 #if defined(LAYER_STATE_8BIT)
 typedef uint8_t layer_state_t;
 #    define MAX_LAYER_BITS 3
@@ -35,13 +58,15 @@ typedef uint16_t layer_state_t;
 #        define MAX_LAYER 16
 #    endif
 #    define get_highest_layer(state) biton16(state)
-#else
+#elif defined(LAYER_STATE_32BIT)
 typedef uint32_t layer_state_t;
 #    define MAX_LAYER_BITS 5
 #    ifndef MAX_LAYER
 #        define MAX_LAYER 32
 #    endif
 #    define get_highest_layer(state) biton32(state)
+#else
+#    error Layer Mask size not specified.  HOW?!
 #endif
 
 /*
@@ -92,7 +117,7 @@ layer_state_t layer_state_set_kb(layer_state_t state);
 
 #    define layer_state_set(layer)
 #    define layer_state_is(layer) (layer == 0)
-#    define layer_state_cmp(state, layer) (state == 0 ? layer == 0 : (state & 1UL << layer) != 0)
+#    define layer_state_cmp(state, layer) (state == 0 ? layer == 0 : (state & (layer_state_t)1 << layer) != 0)
 
 #    define layer_debug()
 #    define layer_clear()