Squashed commit of the following:

commit d614bc5f62f3c2efc5c5cc0f38168a67681e6fb5
Author: Nick Brassel <nick@tzarc.org>
Date:   Sun Oct 16 13:17:03 2022 +1100

    Remove old header generator.

commit 08337b814cfcef57a1f6b41acf06b806ad4bb116
Author: Nick Brassel <nick@tzarc.org>
Date:   Sat Oct 15 11:47:20 2022 +1100

    Restart jinja2 generation for firmware-side output.
This commit is contained in:
Nick Brassel 2022-10-16 13:19:15 +11:00
parent 67e70084ed
commit c2e95c8522
No known key found for this signature in database
8 changed files with 274 additions and 277 deletions

View File

@ -34,10 +34,10 @@ class {{ name }}(namedtuple('{{ name }}', '{{ members }}')):
{{ loop(item.routes.values()) }}
{%- endif -%}
{% if item.request_struct_members %}
# TODO: gen inbound object for {{ to_snake(item.define) }}
# TODO: gen inbound object for {{ item.define | to_snake }}
{%- endif -%}
{% if item.return_struct_members %}
# TODO: gen outbound object for {{ to_snake(item.define) }}
# TODO: gen outbound object for {{ item.define | to_snake }}
{%- endif -%}
{%- endfor %}

View File

@ -0,0 +1,166 @@
{{ GPL2_HEADER_C_LIKE }}
{{ GENERATED_HEADER_C_LIKE }}
#pragma once
#include <stdint.h>
////////////////////////////////////////////////////////////////////////////////
// Versions and identifiers
#define XAP_BCD_VERSION UINT32_C({{ xap.version | triplet_to_bcd }})
#define QMK_BCD_VERSION UINT32_C({{ qmk_version | triplet_to_bcd }})
#define XAP_KEYBOARD_IDENTIFIER UINT32_C({{ keyboard | fnv1a_32 }})
////////////////////////////////////////////////////////////////////////////////
// Response flag definitions
{% for bit,data in xap.response_flags.bits | dictsort -%}
#define {{ xap.response_flags.define_prefix }}_{{ data.define | to_snake | upper }} (UINT32_C(1) << ({{ bit }}))
{% endfor -%}
#define {{ xap.response_flags.define_prefix }}_FAILED 0x00
////////////////////////////////////////////////////////////////////////////////
// Broadcast message definitions
{% for message_id,data in xap.broadcast_messages.messages | dictsort -%}
#define {{ xap.broadcast_messages.define_prefix }}_{{ data.define | to_snake | upper }} {{ message_id }}
{% if 'return_type' in data -%}
void {{ xap.broadcast_messages.define_prefix | lower }}_{{ data.define | to_snake | lower }}({{ data.return_type | type_to_c('value') }});
{% else -%}
void {{ xap.broadcast_messages.define_prefix | lower }}_{{ data.define | to_snake | lower }}(const void *data, size_t length);
{% endif %}
{% endfor -%}
#define XAP_BROADCAST_TOKEN 0xFFFF
////////////////////////////////////////////////////////////////////////////////
// Type definitions
{% for name,data in xap.type_definitions | dictsort -%}
{% if data.type != 'struct' -%}
typedef {{ data.type | type_to_c('xap_'+(name|to_snake|lower)+'_t') }};
{% endif -%}
{% endfor %}
{%- for name,data in xap.type_definitions | dictsort %}
{% if data.type == 'struct' -%}
typedef struct {
{%- for member in data.struct_members %}
{{ member.type | type_to_c(member.name) }};
{%- endfor %}
} __attribute__((__packed__)) xap_{{ name | to_snake | lower }}_t{{ data.type | type_to_c_after }};
_Static_assert(sizeof(xap_{{ name | to_snake | lower }}_t) == {{ data.struct_length }}, "xap_{{ name | to_snake | lower }}_t needs to be {{ data.struct_length }} bytes in size");
{%- endif -%}
{% endfor %}
////////////////////////////////////////////////////////////////////////////////
// Route definitions
{% macro export_route_types(prefix, container) -%}
{%- if 'routes' in container -%}
{% for route, data in container.routes | dictsort -%}
{%- set this_prefix_uc = (prefix + '_' + data.define) | upper -%}
{%- set this_prefix_lc = this_prefix_uc | lower -%}
{% if 'request_struct_members' in data -%}
typedef struct {
{%- for member in data.request_struct_members %}
{{ member.type | type_to_c(member.name|lower) }};
{%- endfor %}
} __attribute__((__packed__)) {{ this_prefix_lc | to_snake | lower }}_arg_t;
_Static_assert(sizeof({{ this_prefix_lc | to_snake | lower }}_arg_t) == {{ data.request_struct_length }}, "{{ this_prefix_lc | to_snake | lower }}_arg_t needs to be {{ data.request_struct_length }} bytes in size");
{% elif 'request_type' in data -%}
typedef {{ data.request_type | type_to_c(this_prefix_lc+'_arg_t') }};
{%- endif -%}
{%- if 'return_struct_members' in data -%}
typedef struct {
{%- for member in data.return_struct_members %}
{{ member.type | type_to_c(member.name|lower) }};
{%- endfor %}
} __attribute__((__packed__)) {{ this_prefix_lc | to_snake | lower }}_t;
_Static_assert(sizeof({{ this_prefix_lc | to_snake | lower }}_t) == {{ data.return_struct_length }}, "{{ this_prefix_lc | to_snake | lower }}_t needs to be {{ data.return_struct_length }} bytes in size");
{%- elif 'return_type' in data -%}
{%- if '[' in data.return_type %}
typedef struct __attribute__((__packed__)) { {{ data.return_type | type_to_c('x') }}; } {{ this_prefix_lc }}_t;
{%- else -%}
typedef {{ data.return_type | type_to_c(this_prefix_lc+'_t') }};
{%- endif -%}
{%- endif %}
{{ export_route_types(this_prefix_lc, data) }}
{% endfor -%}
{%- endif -%}
{%- endmacro -%}
{{ export_route_types('xap_route', xap) }}
////////////////////////////////////////////////////////////////////////////////
// Capabilities IDs
{% macro export_route_ids(prefix, container) -%}
{%- if 'routes' in container -%}
{% for route, data in container.routes | dictsort -%}
{%- set this_prefix_uc = (prefix + '_' + data.define) | upper -%}
{%- set this_prefix_lc = this_prefix_uc | lower -%}
#define {{ this_prefix_uc }} {{ route }}
{{ export_route_ids(this_prefix_uc, data) }}
{%- endfor -%}
{%- endif -%}
{%- endmacro -%}
{{ export_route_ids('XAP_ROUTE', xap) }}
////////////////////////////////////////////////////////////////////////////////
// Capabilities Masks
{% macro export_route_masks(prefix, container, preprocessor_condition) -%}
{%- if 'routes' in container -%}
{% for route, data in container.routes | dictsort -%}
{%- set this_prefix_uc = (prefix + '_' + data.define) | upper -%}
{%- set this_prefix_lc = this_prefix_uc | lower -%}
{% if 'enable_if_preprocessor' in data %}
{% if preprocessor_condition == 'TRUE' %}
{%- set condition = "(" + data.enable_if_preprocessor + ")" -%}
{% else %}
{%- set condition = "(" + preprocessor_condition + " && (" + data.enable_if_preprocessor + "))" -%}
{% endif %}
{% else %}
{%- set condition = preprocessor_condition -%}
{% endif %}
{% if condition == 'TRUE' %}
#define {{ this_prefix_uc }}_MASK (UINT32_C(1) << ({{ this_prefix_uc }}))
{% else %}
#if ({{ condition }})
#define {{ this_prefix_uc }}_MASK (UINT32_C(1) << ({{ this_prefix_uc }}))
#else // ({{ condition }})
#define {{ this_prefix_uc }}_MASK 0
#endif // ({{ condition }})
{% endif %}
{{ export_route_masks(this_prefix_uc, data, condition) }}
{%- endfor -%}
{%- endif -%}
{%- endmacro -%}
{{ export_route_masks('XAP_ROUTE', xap, 'TRUE') }}
////////////////////////////////////////////////////////////////////////////////
// Capabilities Values
{% macro export_route_capabilities(prefix, container) -%}
{%- if 'routes' in container -%}
#define {{ prefix }}_CAPABILITIES (0 \
{% for route, data in container.routes | dictsort -%}
{%- set this_prefix_uc = (prefix + '_' + data.define) | upper -%}
| ({{ this_prefix_uc }}_MASK) \
{% endfor -%}
)
{% for route, data in container.routes | dictsort -%}
{%- set this_prefix_uc = (prefix + '_' + data.define) | upper -%}
{{ export_route_capabilities(this_prefix_uc, data) }}
{% endfor -%}
{%- endif -%}
{%- endmacro -%}
{{ export_route_capabilities('XAP_ROUTE', xap) }}

View File

@ -0,0 +1,9 @@
{{ GPL2_HEADER_C_LIKE }}
{{ GENERATED_HEADER_C_LIKE }}
////////////////////////////////////////////////////////////////////////////////
// Full XAP {{ xap.version }} definitions
#if 0
{{ xap | tojson(4) }}
#endif

View File

@ -4,9 +4,9 @@ from milc import cli
from qmk.path import normpath
from qmk.keyboard import keyboard_completer, keyboard_folder
from qmk.xap.common import render_xap_output
from qmk.xap.gen_firmware.blob_generator import generate_blob
from qmk.xap.gen_firmware.inline_generator import generate_inline
from qmk.xap.gen_firmware.header_generator import generate_header
@cli.argument('-o', '--output', type=normpath, help='File to write to')
@ -28,6 +28,12 @@ def xap_generate_qmk_inc(cli):
generate_inline(cli.args.output, cli.args.keyboard, cli.args.keymap)
with open(normpath(str(cli.args.output.resolve()) + '.generated.j2'), 'w', encoding='utf-8') as out_file:
r = render_xap_output('firmware', 'xap_generated.inl.j2', keyboard=cli.args.keyboard, keymap=cli.args.keymap)
while r.find('\n\n\n') != -1:
r = r.replace('\n\n\n', '\n\n')
out_file.write(r)
@cli.argument('-o', '--output', type=normpath, help='File to write to')
@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='Name of the keyboard')
@ -46,7 +52,11 @@ def xap_generate_qmk_h(cli):
cli.subcommands['xap-generate-qmk-h'].print_help()
return False
generate_header(cli.args.output, cli.args.keyboard, cli.args.keymap)
with open(cli.args.output, 'w', encoding='utf-8') as out_file:
r = render_xap_output('firmware', 'xap_generated.h.j2', keyboard=cli.args.keyboard, keymap=cli.args.keymap)
while r.find('\n\n\n') != -1:
r = r.replace('\n\n\n', '\n\n')
out_file.write(r)
@cli.argument('-o', '--output', type=normpath, help='File to write to')

View File

@ -9,11 +9,13 @@ from typing import OrderedDict
from jinja2 import Environment, FileSystemLoader, select_autoescape
from qmk.casing import to_snake
from qmk.constants import QMK_FIRMWARE
from qmk.constants import QMK_FIRMWARE, GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE
from qmk.git import git_get_version
from qmk.json_schema import json_load, validate
from qmk.decorators import lru_cache
from qmk.keymap import locate_keymap
from qmk.path import keyboard
from qmk.xap.jinja2_filters import attach_filters
XAP_SPEC = 'xap.hjson'
@ -57,16 +59,18 @@ def _get_jinja2_env(data_templates_xap_subdir: str):
return j2
def render_xap_output(data_templates_xap_subdir, file_to_render, defs):
def render_xap_output(data_templates_xap_subdir, file_to_render, defs=None, **kwargs):
if defs is None:
defs = latest_xap_defs()
j2 = _get_jinja2_env(data_templates_xap_subdir)
j2.globals['to_snake'] = to_snake
attach_filters(j2)
constants = {}
for feature in ['rgblight', 'rgb_matrix', 'led_matrix']:
constants[feature] = load_lighting_spec(feature)
return j2.get_template(file_to_render).render(xap=defs, xap_str=hjson.dumps(defs), constants=constants)
return j2.get_template(file_to_render).render(xap=defs, qmk_version=git_get_version(), xap_str=hjson.dumps(defs), constants=constants, GPL2_HEADER_C_LIKE=GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE=GENERATED_HEADER_C_LIKE, **kwargs)
def _find_kb_spec(kb):

View File

@ -7,7 +7,7 @@ from pathlib import Path
from qmk.info import keymap_json
from qmk.commands import get_chunks, dump_lines
from qmk.json_schema import deep_update, json_load
from qmk.json_encoders import InfoJSONEncoder
from qmk.constants import GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE
@ -57,6 +57,12 @@ def generate_blob(output_file, keyboard, keymap):
lines = [GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE, '#pragma once', '']
lines.append(f'#if 0')
lines.append('// Blob contains a minified+gzipped version of the following:')
lines.append(json.dumps(info_json, cls=InfoJSONEncoder))
lines.append(f'#endif')
lines.append('')
# Gen output file
lines.append('static const unsigned char config_blob_gz[] PROGMEM = {')
lines.append(data)

View File

@ -1,268 +0,0 @@
"""This script generates the XAP protocol generated header to be compiled into QMK.
"""
import re
from fnvhash import fnv1a_32
from qmk.casing import to_snake
from qmk.commands import dump_lines
from qmk.git import git_get_version
from qmk.constants import GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE
from qmk.xap.common import merge_xap_defs, route_conditions
def _get_c_type(xap_type):
if xap_type == 'bool':
return 'bool'
elif xap_type == 'u8':
return 'uint8_t'
elif xap_type == 'u16':
return 'uint16_t'
elif xap_type == 'u32':
return 'uint32_t'
elif xap_type == 'u64':
return 'uint64_t'
elif xap_type == 'struct':
return 'struct'
elif xap_type == 'string':
return 'const char *'
return 'unknown'
def _append_route_defines(lines, container, container_id=None, route_stack=None):
"""Handles building the list of the XAP routes, combining parent and child names together, as well as the route number.
"""
if route_stack is None:
route_stack = [container]
else:
route_stack.append(container)
route_name = '_'.join([r['define'] for r in route_stack])
if container_id:
lines.append(f'#define {route_name} {container_id}')
if 'routes' in container:
for route_id in container['routes']:
route = container['routes'][route_id]
_append_route_defines(lines, route, route_id, route_stack)
route_stack.pop()
def _append_route_masks(lines, container, container_id=None, route_stack=None):
"""Handles creating the equivalent XAP route masks, for capabilities checks. Forces value of `0` if disabled in the firmware.
"""
if route_stack is None:
route_stack = [container]
else:
route_stack.append(container)
route_name = '_'.join([r['define'] for r in route_stack])
condition = route_conditions(route_stack)
if container_id:
if condition:
lines.append('')
lines.append(f'#if {condition}')
lines.append(f'#define {route_name}_MASK (1ul << ({route_name}))')
if condition:
lines.append(f'#else // {condition}')
lines.append(f'#define {route_name}_MASK 0')
lines.append(f'#endif // {condition}')
lines.append('')
if 'routes' in container:
for route_id in container['routes']:
route = container['routes'][route_id]
_append_route_masks(lines, route, route_id, route_stack)
route_stack.pop()
def _append_route_capabilities(lines, container, container_id=None, route_stack=None):
"""Handles creating the equivalent XAP route masks, for capabilities checks. Forces value of `0` if disabled in the firmware.
"""
if route_stack is None:
route_stack = [container]
else:
route_stack.append(container)
route_name = '_'.join([r['define'] for r in route_stack])
if 'routes' in container:
lines.append('')
lines.append(f'#define {route_name}_CAPABILITIES (0 \\')
if 'routes' in container:
for route_id in container['routes']:
route = container['routes'][route_id]
route_stack.append(route)
child_name = '_'.join([r['define'] for r in route_stack])
lines.append(f' | ({child_name}_MASK) \\')
route_stack.pop()
lines.append(' )')
if 'routes' in container:
for route_id in container['routes']:
route = container['routes'][route_id]
_append_route_capabilities(lines, route, route_id, route_stack)
route_stack.pop()
def _append_route_types(lines, container, container_id=None, route_stack=None):
"""Handles creating typedefs used by routes
"""
if route_stack is None:
route_stack = [container]
else:
route_stack.append(container)
route_name = to_snake('_'.join([r['define'] for r in route_stack]))
# Inbound
if 'request_struct_members' in container:
request_struct_members = container['request_struct_members']
lines.append('typedef struct {')
for member in request_struct_members:
member_type = _get_c_type(member['type'])
member_name = to_snake(member['name'])
lines.append(f' {member_type} {member_name};')
lines.append(f'}} __attribute__((__packed__)) {route_name}_arg_t;')
req_len = container['request_struct_length']
lines.append(f'_Static_assert(sizeof({route_name}_arg_t) == {req_len}, "{route_name}_arg_t needs to be {req_len} bytes in size");')
elif 'request_type' in container:
request_type = container['request_type']
found = re.search(r'(u\d+)\[(\d+)\]', request_type)
if found:
request_type, size = found.groups()
lines.append(f'typedef struct __attribute__((__packed__)) {{ {_get_c_type(request_type)} x[{size}]; }} {route_name}_arg_t;')
else:
lines.append(f'typedef {_get_c_type(request_type)} {route_name}_arg_t;')
# Outbound
qualifier = 'const' if 'return_constant' in container else ''
if 'return_struct_members' in container:
return_struct_members = container['return_struct_members']
lines.append('typedef struct {')
for member in return_struct_members:
member_type = _get_c_type(member['type'])
member_name = f'{qualifier} {to_snake(member["name"])}'
lines.append(f' {member_type} {member_name};')
lines.append(f'}} __attribute__((__packed__)) {route_name}_t;')
req_len = container['return_struct_length']
lines.append(f'_Static_assert(sizeof({route_name}_t) == {req_len}, "{route_name}_t needs to be {req_len} bytes in size");')
elif 'return_type' in container:
return_type = container['return_type']
found = re.search(r'(u\d+)\[(\d+)\]', return_type)
if found:
return_type, size = found.groups()
lines.append(f'typedef struct __attribute__((__packed__)) {{ {_get_c_type(return_type)} x[{size}]; }} {route_name}_t;')
else:
lines.append(f'typedef {_get_c_type(return_type)} {route_name}_t;')
# Recurse
if 'routes' in container:
for route_id in container['routes']:
route = container['routes'][route_id]
_append_route_types(lines, route, route_id, route_stack)
route_stack.pop()
def _append_internal_types(lines, container):
"""Handles creating the various constants, types, defines, etc.
"""
response_flags = container.get('response_flags', {})
prefix = response_flags['define_prefix']
for key, value in response_flags['bits'].items():
define = value.get('define')
lines.append(f'#define {prefix}_{define} (1ul << ({key}))')
# Add special
lines.append(f'#define {prefix}_FAILED 0x00')
lines.append('')
broadcast_messages = container.get('broadcast_messages', {})
broadcast_prefix = broadcast_messages['define_prefix']
for key, value in broadcast_messages['messages'].items():
define = value.get('define')
name = to_snake(f'{broadcast_prefix}_{define}')
lines.append(f'#define {broadcast_prefix}_{define} {key}')
if 'return_type' in value:
ret_type = _get_c_type(value['return_type'])
lines.append(f'void {name}({ret_type} value);')
else:
lines.append(f'void {name}(const void *data, size_t length);')
# Add special
lines.append(f'#define {broadcast_prefix}_TOKEN 0xFFFF')
lines.append('')
additional_types = {}
types = container.get('type_definitions', {})
for key, value in types.items():
data_type = _get_c_type(value['type'])
additional_types[key] = f'xap_{key}_t'
for key, value in types.items():
data_type = _get_c_type(value['type'])
if data_type == 'struct':
members = value['struct_members']
lines.append(f'typedef {data_type} {{')
for member in members:
member_name = member["name"]
member_type = _get_c_type(member["type"])
if member_type == 'unknown':
member_type = additional_types[member["type"]]
lines.append(f' {member_type} {member_name};')
lines.append(f'}} __attribute__((__packed__)) xap_{key}_t;')
req_len = value['struct_length']
lines.append(f'_Static_assert(sizeof(xap_{key}_t) == {req_len}, "xap_{key}_t needs to be {req_len} bytes in size");')
else:
lines.append(f'typedef {data_type} xap_{key}_t;')
def generate_header(output_file, keyboard, keymap):
"""Generates the XAP protocol header file, generated during normal build.
"""
xap_defs = merge_xap_defs(keyboard, keymap)
# Preamble
lines = [GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE, '#pragma once', '']
# Versions
prog = re.compile(r'^(\d+)\.(\d+)\.(\d+)')
b = prog.match(xap_defs['version'])
lines.append(f'#define XAP_BCD_VERSION 0x{int(b.group(1)):02X}{int(b.group(2)):02X}{int(b.group(3)):04X}ul')
b = prog.findall(git_get_version() or "") or [('0', '0', '0')]
lines.append(f'#define QMK_BCD_VERSION 0x{int(b[0][0]):02X}{int(b[0][1]):02X}{int(b[0][2]):04X}ul')
keyboard_id = fnv1a_32(bytes(keyboard, 'utf-8'))
lines.append(f'#define XAP_KEYBOARD_IDENTIFIER 0x{keyboard_id:08X}ul')
lines.append('')
# Types
_append_internal_types(lines, xap_defs)
lines.append('')
_append_route_types(lines, xap_defs)
lines.append('')
# Append the route and command defines
_append_route_defines(lines, xap_defs)
lines.append('')
_append_route_masks(lines, xap_defs)
lines.append('')
_append_route_capabilities(lines, xap_defs)
lines.append('')
dump_lines(output_file, lines)

View File

@ -0,0 +1,70 @@
"""This script enables attachment of XAP-specific filters to Jinja2
"""
import re
from fnvhash import fnv1a_32
from jinja2 import Environment
from qmk.casing import to_snake
TRIPLET_PATTERN = re.compile(r'^(\d+)\.(\d+)\.(\d+)')
TYPE_ARRAY_PATTERN = re.compile(r'^([^\[]+)\[([^[\]]+)\]$')
def _fnv1a_32(s: str):
res = fnv1a_32(bytes(s, 'utf-8'))
return f'0x{res:08X}'
def _xap_type_to_c_before(xt: str):
m = TYPE_ARRAY_PATTERN.match(xt)
if m:
return _xap_type_to_c(m.group(1))
if xt == 'u8':
return 'uint8_t'
elif xt == 'u16':
return 'uint16_t'
elif xt == 'u32':
return 'uint32_t'
elif xt == 'u64':
return 'uint64_t'
elif xt == 'string':
return 'const char*'
elif xt == 'token':
return 'xap_token_t'
elif xt == 'response_flags':
return 'xap_response_flags_t'
raise TypeError(f'Unknown XAP type: {xt}')
def _xap_type_to_c_after(xt: str):
m = TYPE_ARRAY_PATTERN.match(xt)
if m:
try:
extent = int(m.group(2))
return f'[{extent}]'
except:
return '[]'
return ''
def _xap_type_to_c(xt: str, name: str = None):
if name is not None:
name = re.sub(' ', '_', name)
return f'{(_xap_type_to_c_before(xt))} {name}{(_xap_type_to_c_after(xt))}'
return f'{(_xap_type_to_c_before(xt))}{(_xap_type_to_c_after(xt))}'
def _triplet_to_bcd(value: str):
m = TRIPLET_PATTERN.match(value)
if not m:
return '0'
return f'0x{int(m.group(1)):02d}{int(m.group(2)):02d}{int(m.group(3)):04d}'
def attach_filters(j2: Environment):
j2.filters['to_snake'] = to_snake
j2.filters['triplet_to_bcd'] = _triplet_to_bcd
j2.filters['fnv1a_32'] = _fnv1a_32
j2.filters['type_to_c'] = _xap_type_to_c
j2.filters['type_to_c_before'] = _xap_type_to_c_before
j2.filters['type_to_c_after'] = _xap_type_to_c_after