Merge remote-tracking branch 'origin/develop' into xap

docs/cli_commands.md

@@ -165,16 +165,31 @@ qmk find -f 'processor=STM32F411'
 qmk find -f 'processor=STM32F411' -f 'features.rgb_matrix=true'
 ```
 
+The following filter expressions are also supported:
+
+ - `exists(key)`: Match targets where `key` is present.
+ - `absent(key)`: Match targets where `key` is not present.
+ - `contains(key, value)`: Match targets where `key` contains `value`. Can be used for strings, arrays and object keys.
+ - `length(key, value)`: Match targets where the length of `key` is `value`. Can be used for strings, arrays and objects.
+
+You can also list arbitrary values for each matched target with `--print`:
+
+```
+qmk find -f 'processor=STM32F411' -p 'keyboard_name' -p 'features.rgb_matrix'
+```
+
 **Usage**:
 
 ```
-qmk find [-h] [-km KEYMAP] [-f FILTER]
+qmk find [-h] [-km KEYMAP] [-p PRINT] [-f FILTER]
 
 options:
   -km KEYMAP, --keymap KEYMAP
                         The keymap name to build. Default is 'default'.
+  -p PRINT, --print PRINT
+                        For each matched target, print the value of the supplied info.json key. May be passed multiple times.
   -f FILTER, --filter FILTER
-                        Filter the list of keyboards based on the supplied value in rules.mk. Matches info.json structure, and accepts the formats 'features.rgblight=true' or 'exists(matrix_pins.direct)'. May be passed multiple times, all filters need to match. Value may include wildcards such as '*' and '?'.
+                        Filter the list of keyboards based on their info.json data. Accepts the formats key=value, function(key), or function(key,value), eg. 'features.rgblight=true'. Valid functions are 'absent', 'contains', 'exists' and 'length'. May be passed multiple times; all filters need to match. Value may include wildcards such as '*' and '?'.
 ```
 
 ## `qmk console`

docs/understanding_qmk.md

@@ -129,9 +129,9 @@ The `process_record()` function itself is deceptively simple, but hidden within
 
 * [`void action_exec(keyevent_t event)`](https://github.com/qmk/qmk_firmware/blob/325da02e57fe7374e77b82cb00360ba45167e25c/quantum/action.c#L78-L140)
     * [`void pre_process_record_quantum(keyrecord_t *record)`](https://github.com/qmk/qmk_firmware/blob/325da02e57fe7374e77b82cb00360ba45167e25c/quantum/quantum.c#L204)
-      * [`bool process_combo(uint16_t keycode, keyrecord_t *record)`](https://github.com/qmk/qmk_firmware/blob/325da02e57fe7374e77b82cb00360ba45167e25c/quantum/process_keycode/process_combo.c#L521)
       * [`bool pre_process_record_kb(uint16_t keycode, keyrecord_t *record)`](https://github.com/qmk/qmk_firmware/blob/27119fa77e8a1b95fff80718d3db4f3e32849298/quantum/quantum.c#L117)
         * [`bool pre_process_record_user(uint16_t keycode, keyrecord_t *record)`](https://github.com/qmk/qmk_firmware/blob/27119fa77e8a1b95fff80718d3db4f3e32849298/quantum/quantum.c#L121)
+      * [`bool process_combo(uint16_t keycode, keyrecord_t *record)`](https://github.com/qmk/qmk_firmware/blob/325da02e57fe7374e77b82cb00360ba45167e25c/quantum/process_keycode/process_combo.c#L521)
   * [`void process_record(keyrecord_t *record)`](https://github.com/qmk/qmk_firmware/blob/325da02e57fe7374e77b82cb00360ba45167e25c/quantum/action.c#L254)
     * [`bool process_record_quantum(keyrecord_t *record)`](https://github.com/qmk/qmk_firmware/blob/325da02e57fe7374e77b82cb00360ba45167e25c/quantum/quantum.c#L224)
       * [Map this record to a keycode](https://github.com/qmk/qmk_firmware/blob/325da02e57fe7374e77b82cb00360ba45167e25c/quantum/quantum.c#L225)

lib/python/qmk/cli/c2json.py

@@ -57,7 +57,7 @@ def c2json(cli):
         cli.args.output.parent.mkdir(parents=True, exist_ok=True)
         if cli.args.output.exists():
             cli.args.output.replace(cli.args.output.parent / (cli.args.output.name + '.bak'))
-        cli.args.output.write_text(json.dumps(keymap_json, cls=InfoJSONEncoder))
+        cli.args.output.write_text(json.dumps(keymap_json, cls=InfoJSONEncoder, sort_keys=True))
 
         if not cli.args.quiet:
             cli.log.info('Wrote keymap to %s.', cli.args.output)

lib/python/qmk/cli/find.py

@@ -11,13 +11,17 @@ from qmk.search import search_keymap_targets
     action='append',
     default=[],
     help=  # noqa: `format-python` and `pytest` don't agree here.
-    "Filter the list of keyboards based on the supplied value in rules.mk. Matches info.json structure, and accepts the formats 'features.rgblight=true' or 'exists(matrix_pins.direct)'. May be passed multiple times, all filters need to match. Value may include wildcards such as '*' and '?'."  # noqa: `format-python` and `pytest` don't agree here.
+    "Filter the list of keyboards based on their info.json data. Accepts the formats key=value, function(key), or function(key,value), eg. 'features.rgblight=true'. Valid functions are 'absent', 'contains', 'exists' and 'length'. May be passed multiple times; all filters need to match. Value may include wildcards such as '*' and '?'."  # noqa: `format-python` and `pytest` don't agree here.
 )
+@cli.argument('-p', '--print', arg_only=True, action='append', default=[], help="For each matched target, print the value of the supplied info.json key. May be passed multiple times.")
 @cli.argument('-km', '--keymap', type=str, default='default', help="The keymap name to build. Default is 'default'.")
 @cli.subcommand('Find builds which match supplied search criteria.')
 def find(cli):
     """Search through all keyboards and keymaps for a given search criteria.
     """
-    targets = search_keymap_targets(cli.args.keymap, cli.args.filter)
+    targets = search_keymap_targets(cli.args.keymap, cli.args.filter, cli.args.print)
-    for target in targets:
+    for keyboard, keymap, print_vals in targets:
-        print(f'{target[0]}:{target[1]}')
+        print(f'{keyboard}:{keymap}')
+
+        for key, val in print_vals:
+            print(f'    {key}={val}')

lib/python/qmk/cli/format/json.py

@@ -62,4 +62,4 @@ def format_json(cli):
                 json_file['layers'][layer_num] = current_layer
 
     # Display the results
-    print(json.dumps(json_file, cls=json_encoder))
+    print(json.dumps(json_file, cls=json_encoder, sort_keys=True))

lib/python/qmk/cli/generate/info_json.py

@@ -76,7 +76,7 @@ def generate_info_json(cli):
     # Build the info.json file
     kb_info_json = info_json(cli.config.generate_info_json.keyboard)
     strip_info_json(kb_info_json)
-    info_json_text = json.dumps(kb_info_json, indent=4, cls=InfoJSONEncoder)
+    info_json_text = json.dumps(kb_info_json, indent=4, cls=InfoJSONEncoder, sort_keys=True)
 
     if cli.args.output:
         # Write to a file

lib/python/qmk/cli/info.py

@@ -200,7 +200,7 @@ def info(cli):
 
     # Output in the requested format
     if cli.args.format == 'json':
-        print(json.dumps(kb_info_json, cls=InfoJSONEncoder))
+        print(json.dumps(kb_info_json, cls=InfoJSONEncoder, sort_keys=True))
         return True
     elif cli.args.format == 'text':
         print_dotted_output(kb_info_json)

lib/python/qmk/cli/migrate.py

@@ -75,7 +75,7 @@ def migrate(cli):
 
     # Finally write out updated info.json
     cli.log.info(f'  Updating {target_info}')
-    target_info.write_text(json.dumps(info_data.to_dict(), cls=InfoJSONEncoder))
+    target_info.write_text(json.dumps(info_data.to_dict(), cls=InfoJSONEncoder, sort_keys=True))
 
     cli.log.info(f'{{fg_green}}Migration of keyboard {{fg_cyan}}{cli.args.keyboard}{{fg_green}} complete!{{fg_reset}}')
     cli.log.info(f"Verify build with {{fg_yellow}}qmk compile -kb {cli.args.keyboard} -km default{{fg_reset}}.")

lib/python/qmk/cli/new/keyboard.py

@@ -102,7 +102,7 @@ def augment_community_info(src, dest):
         item["matrix"] = [int(item["y"]), int(item["x"])]
 
     # finally write out the updated info.json
-    dest.write_text(json.dumps(info, cls=InfoJSONEncoder))
+    dest.write_text(json.dumps(info, cls=InfoJSONEncoder, sort_keys=True))
 
 
 def _question(*args, **kwargs):

lib/python/qmk/cli/via2json.py

@@ -141,5 +141,5 @@ def via2json(cli):
     # Generate the keymap.json
     keymap_json = generate_json(cli.args.keymap, cli.args.keyboard, keymap_layout, keymap_data, macro_data)
 
-    keymap_lines = [json.dumps(keymap_json, cls=KeymapJSONEncoder)]
+    keymap_lines = [json.dumps(keymap_json, cls=KeymapJSONEncoder, sort_keys=True)]
     dump_lines(cli.args.output, keymap_lines, cli.args.quiet)

lib/python/qmk/importers.py

@@ -91,7 +91,7 @@ def import_keymap(keymap_data):
     keyboard_keymap.parent.mkdir(parents=True, exist_ok=True)
 
     # Dump out all those lovely files
-    keyboard_keymap.write_text(json.dumps(keymap_data, cls=KeymapJSONEncoder))
+    keyboard_keymap.write_text(json.dumps(keymap_data, cls=KeymapJSONEncoder, sort_keys=True))
 
     return (kb_name, km_name)
 
@@ -139,8 +139,8 @@ def import_keyboard(info_data, keymap_data=None):
     temp = json_load(keyboard_info)
     deep_update(temp, info_data)
 
-    keyboard_info.write_text(json.dumps(temp, cls=InfoJSONEncoder))
+    keyboard_info.write_text(json.dumps(temp, cls=InfoJSONEncoder, sort_keys=True))
-    keyboard_keymap.write_text(json.dumps(keymap_data, cls=KeymapJSONEncoder))
+    keyboard_keymap.write_text(json.dumps(keymap_data, cls=KeymapJSONEncoder, sort_keys=True))
 
     return kb_name
 

lib/python/qmk/json_encoders.py

@@ -27,39 +27,56 @@ class QMKJSONEncoder(json.JSONEncoder):
 
         return float(obj)
 
-    def encode_dict_single_line(self, obj):
+    def encode_dict(self, obj, path):
-        return "{" + ", ".join(f"{self.encode(key)}: {self.encode(element)}" for key, element in sorted(obj.items(), key=self.sort_layout)) + "}"
+        """Encode a dict-like object.
+        """
+        if obj:
+            self.indentation_level += 1
 
-    def encode_list(self, obj, key=None):
+            items = sorted(obj.items(), key=self.sort_dict) if self.sort_keys else obj.items()
+            output = [self.indent_str + f"{json.dumps(key)}: {self.encode(value, path + [key])}" for key, value in items]
+
+            self.indentation_level -= 1
+
+            return "{\n" + ",\n".join(output) + "\n" + self.indent_str + "}"
+        else:
+            return "{}"
+
+    def encode_dict_single_line(self, obj, path):
+        """Encode a dict-like object onto a single line.
+        """
+        return "{" + ", ".join(f"{json.dumps(key)}: {self.encode(value, path + [key])}" for key, value in sorted(obj.items(), key=self.sort_layout)) + "}"
+
+    def encode_list(self, obj, path):
         """Encode a list-like object.
         """
         if self.primitives_only(obj):
-            return "[" + ", ".join(self.encode(element) for element in obj) + "]"
+            return "[" + ", ".join(self.encode(value, path + [index]) for index, value in enumerate(obj)) + "]"
 
         else:
             self.indentation_level += 1
 
-            if key in ('layout', 'rotary'):
+            if path[-1] in ('layout', 'rotary'):
-                # These are part of a layout or led/encoder config, put them on a single line.
+                # These are part of a LED layout or encoder config, put them on a single line
-                output = [self.indent_str + self.encode_dict_single_line(element) for element in obj]
+                output = [self.indent_str + self.encode_dict_single_line(value, path + [index]) for index, value in enumerate(obj)]
             else:
-                output = [self.indent_str + self.encode(element) for element in obj]
+                output = [self.indent_str + self.encode(value, path + [index]) for index, value in enumerate(obj)]
 
             self.indentation_level -= 1
 
             return "[\n" + ",\n".join(output) + "\n" + self.indent_str + "]"
 
-    def encode(self, obj, key=None):
+    def encode(self, obj, path=[]):
-        """Encode keymap.json objects for QMK.
+        """Encode JSON objects for QMK.
         """
         if isinstance(obj, Decimal):
             return self.encode_decimal(obj)
 
         elif isinstance(obj, (list, tuple)):
-            return self.encode_list(obj, key)
+            return self.encode_list(obj, path)
 
         elif isinstance(obj, dict):
-            return self.encode_dict(obj, key)
+            return self.encode_dict(obj, path)
 
         else:
             return super().encode(obj)
@@ -80,19 +97,10 @@ class QMKJSONEncoder(json.JSONEncoder):
 class InfoJSONEncoder(QMKJSONEncoder):
     """Custom encoder to make info.json's a little nicer to work with.
     """
-    def encode_dict(self, obj, key):
+    def sort_layout(self, item):
-        """Encode info.json dictionaries.
+        """Sorts the hashes in a nice way.
         """
-        if obj:
+        key = item[0]
-            self.indentation_level += 1
-            output = [self.indent_str + f"{json.dumps(k)}: {self.encode(v, k)}" for k, v in sorted(obj.items(), key=self.sort_dict)]
-            self.indentation_level -= 1
-            return "{\n" + ",\n".join(output) + "\n" + self.indent_str + "}"
-        else:
-            return "{}"
-
-    def sort_layout(self, key):
-        key = key[0]
 
         if key == 'label':
             return '00label'
@@ -117,14 +125,14 @@ class InfoJSONEncoder(QMKJSONEncoder):
 
         return key
 
-    def sort_dict(self, key):
+    def sort_dict(self, item):
         """Forces layout to the back of the sort order.
         """
-        key = key[0]
+        key = item[0]
 
         if self.indentation_level == 1:
             if key == 'manufacturer':
-                return '10keyboard_name'
+                return '10manufacturer'
 
             elif key == 'keyboard_name':
                 return '11keyboard_name'
@@ -150,19 +158,7 @@ class InfoJSONEncoder(QMKJSONEncoder):
 class KeymapJSONEncoder(QMKJSONEncoder):
     """Custom encoder to make keymap.json's a little nicer to work with.
     """
-    def encode_dict(self, obj, key):
+    def encode_list(self, obj, path):
-        """Encode dictionary objects for keymap.json.
-        """
-        if obj:
-            self.indentation_level += 1
-            output = [self.indent_str + f"{json.dumps(k)}: {self.encode(v, k)}" for k, v in sorted(obj.items(), key=self.sort_dict)]
-            self.indentation_level -= 1
-            return "{\n" + ",\n".join(output) + "\n" + self.indent_str + "}"
-
-        else:
-            return "{}"
-
-    def encode_list(self, obj, k=None):
         """Encode a list-like object.
         """
         if self.indentation_level == 2:
@@ -196,10 +192,10 @@ class KeymapJSONEncoder(QMKJSONEncoder):
 
             return "[\n" + ",\n".join(output) + "\n" + self.indent_str + "]"
 
-    def sort_dict(self, key):
+    def sort_dict(self, item):
         """Sorts the hashes in a nice way.
         """
-        key = key[0]
+        key = item[0]
 
         if self.indentation_level == 1:
             if key == 'version':

lib/python/qmk/makefile.py

@@ -18,7 +18,7 @@ def parse_rules_mk_file(file, rules_mk=None):
 
     file = Path(file)
     if file.exists():
-        rules_mk_lines = file.read_text().split("\n")
+        rules_mk_lines = file.read_text(encoding='utf-8').split("\n")
 
         for line in rules_mk_lines:
             # Filter out comments

lib/python/qmk/search.py

@@ -45,7 +45,7 @@ def _load_keymap_info(keyboard, keymap):
         return (keyboard, keymap, keymap_json(keyboard, keymap))
 
 
-def search_keymap_targets(keymap='default', filters=[]):
+def search_keymap_targets(keymap='default', filters=[], print_vals=[]):
     targets = []
 
     with multiprocessing.Pool() as pool:
@@ -66,14 +66,43 @@ def search_keymap_targets(keymap='default', filters=[]):
             cli.log.info('Parsing data for all matching keyboard/keymap combinations...')
             valid_keymaps = [(e[0], e[1], dotty(e[2])) for e in pool.starmap(_load_keymap_info, target_list)]
 
+            function_re = re.compile(r'^(?P<function>[a-zA-Z]+)\((?P<key>[a-zA-Z0-9_\.]+)(,\s*(?P<value>[^#]+))?\)$')
             equals_re = re.compile(r'^(?P<key>[a-zA-Z0-9_\.]+)\s*=\s*(?P<value>[^#]+)$')
-            exists_re = re.compile(r'^exists\((?P<key>[a-zA-Z0-9_\.]+)\)$')
+
-            for filter_txt in filters:
+            for filter_expr in filters:
-                f = equals_re.match(filter_txt)
+                function_match = function_re.match(filter_expr)
-                if f is not None:
+                equals_match = equals_re.match(filter_expr)
-                    key = f.group('key')
+
-                    value = f.group('value')
+                if function_match is not None:
-                    cli.log.info(f'Filtering on condition ("{key}" == "{value}")...')
+                    func_name = function_match.group('function').lower()
+                    key = function_match.group('key')
+                    value = function_match.group('value')
+
+                    if value is not None:
+                        if func_name == 'length':
+                            valid_keymaps = filter(lambda e: key in e[2] and len(e[2].get(key)) == int(value), valid_keymaps)
+                        elif func_name == 'contains':
+                            valid_keymaps = filter(lambda e: key in e[2] and value in e[2].get(key), valid_keymaps)
+                        else:
+                            cli.log.warning(f'Unrecognized filter expression: {function_match.group(0)}')
+                            continue
+
+                        cli.log.info(f'Filtering on condition: {{fg_green}}{func_name}{{fg_reset}}({{fg_cyan}}{key}{{fg_reset}}, {{fg_cyan}}{value}{{fg_reset}})...')
+                    else:
+                        if func_name == 'exists':
+                            valid_keymaps = filter(lambda e: key in e[2], valid_keymaps)
+                        elif func_name == 'absent':
+                            valid_keymaps = filter(lambda e: key not in e[2], valid_keymaps)
+                        else:
+                            cli.log.warning(f'Unrecognized filter expression: {function_match.group(0)}')
+                            continue
+
+                        cli.log.info(f'Filtering on condition: {{fg_green}}{func_name}{{fg_reset}}({{fg_cyan}}{key}{{fg_reset}})...')
+
+                elif equals_match is not None:
+                    key = equals_match.group('key')
+                    value = equals_match.group('value')
+                    cli.log.info(f'Filtering on condition: {{fg_cyan}}{key}{{fg_reset}} == {{fg_cyan}}{value}{{fg_reset}}...')
 
                     def _make_filter(k, v):
                         expr = fnmatch.translate(v)
@@ -87,13 +116,10 @@ def search_keymap_targets(keymap='default', filters=[]):
                         return f
 
                     valid_keymaps = filter(_make_filter(key, value), valid_keymaps)
+                else:
+                    cli.log.warning(f'Unrecognized filter expression: {filter_expr}')
+                    continue
 
-                f = exists_re.match(filter_txt)
+            targets = [(e[0], e[1], [(p, e[2].get(p)) for p in print_vals]) for e in valid_keymaps]
-                if f is not None:
-                    key = f.group('key')
-                    cli.log.info(f'Filtering on condition (exists: "{key}")...')
-                    valid_keymaps = filter(lambda e: e[2].get(key) is not None, valid_keymaps)
-
-            targets = [(e[0], e[1]) for e in valid_keymaps]
 
     return targets

quantum/quantum.c

@@ -211,12 +211,11 @@ uint16_t get_event_keycode(keyevent_t event, bool update_layer_cache) {
 /* Get keycode, and then process pre tapping functionality */
 bool pre_process_record_quantum(keyrecord_t *record) {
     uint16_t keycode = get_record_keycode(record, true);
+    return pre_process_record_kb(keycode, record) &&
 #ifdef COMBO_ENABLE
-    if (!(process_combo(keycode, record))) {
+           process_combo(keycode, record) &&
-        return false;
-    }
 #endif
-    return pre_process_record_kb(keycode, record);
+           true;
 }
 
 /* Get keycode, and then call keyboard function */