2020-10-25 21:48:44 +00:00
""" This script automates the generation of the QMK API data.
"""
from pathlib import Path
2022-02-27 11:28:51 +00:00
import shutil
2020-10-25 21:48:44 +00:00
import json
from milc import cli
2023-12-06 04:55:35 +00:00
import qmk . path
2020-10-25 21:48:44 +00:00
from qmk . datetime import current_datetime
from qmk . info import info_json
2021-03-24 16:26:38 +00:00
from qmk . json_schema import json_load
2023-01-11 01:38:35 +00:00
from qmk . keymap import list_keymaps
2023-11-22 00:14:34 +00:00
from qmk . keyboard import find_readme , list_keyboards , keyboard_alias_definitions
2022-12-09 00:54:52 +00:00
from qmk . keycodes import load_spec , list_versions , list_languages
2020-10-25 21:48:44 +00:00
2022-08-06 06:14:29 +00:00
DATA_PATH = Path ( ' data ' )
TEMPLATE_PATH = DATA_PATH / ' templates/api/ '
2022-02-27 11:28:51 +00:00
BUILD_API_PATH = Path ( ' .build/api_data/ ' )
2020-10-25 21:48:44 +00:00
2022-11-23 18:01:07 +00:00
def _list_constants ( output_folder ) :
""" Produce a map of available constants
"""
ret = { }
for file in ( output_folder / ' constants ' ) . glob ( ' **/*_[0-9].[0-9].[0-9].json ' ) :
name , version = file . stem . rsplit ( ' _ ' , 1 )
if name not in ret :
ret [ name ] = [ ]
ret [ name ] . append ( version )
# Ensure content is sorted
for name in ret :
ret [ name ] = sorted ( ret [ name ] )
return ret
2022-11-05 10:30:09 +00:00
def _resolve_keycode_specs ( output_folder ) :
""" To make it easier for consumers, publish pre-merged spec files
"""
for version in list_versions ( ) :
overall = load_spec ( version )
output_file = output_folder / f ' constants/keycodes_ { version } .json '
2023-03-29 12:01:16 +00:00
output_file . write_text ( json . dumps ( overall , separators = ( ' , ' , ' : ' ) ) , encoding = ' utf-8 ' )
2022-11-05 10:30:09 +00:00
2022-12-09 00:54:52 +00:00
for lang in list_languages ( ) :
for version in list_versions ( lang ) :
overall = load_spec ( version , lang )
output_file = output_folder / f ' constants/keycodes_ { lang } _ { version } .json '
2023-03-29 12:01:16 +00:00
output_file . write_text ( json . dumps ( overall , separators = ( ' , ' , ' : ' ) ) , encoding = ' utf-8 ' )
2022-12-09 00:54:52 +00:00
2022-11-08 03:03:02 +00:00
# Purge files consumed by 'load_spec'
shutil . rmtree ( output_folder / ' constants/keycodes/ ' )
def _filtered_copy ( src , dst ) :
src = Path ( src )
dst = Path ( dst )
if dst . suffix == ' .hjson ' :
data = json_load ( src )
dst = dst . with_suffix ( ' .json ' )
2023-03-29 12:01:16 +00:00
dst . write_text ( json . dumps ( data , separators = ( ' , ' , ' : ' ) ) , encoding = ' utf-8 ' )
2022-11-08 03:03:02 +00:00
return dst
2023-03-27 19:01:07 +00:00
if dst . suffix == ' .jsonschema ' :
data = json_load ( src )
dst . write_text ( json . dumps ( data ) , encoding = ' utf-8 ' )
return dst
2022-11-08 03:03:02 +00:00
return shutil . copy2 ( src , dst )
2022-11-05 10:30:09 +00:00
2022-08-06 06:14:29 +00:00
def _filtered_keyboard_list ( ) :
""" Perform basic filtering of list_keyboards
"""
keyboard_list = list_keyboards ( )
if cli . args . filter :
kb_list = [ ]
for keyboard_name in keyboard_list :
if any ( i in keyboard_name for i in cli . args . filter ) :
kb_list . append ( keyboard_name )
keyboard_list = kb_list
return keyboard_list
2021-05-10 18:18:44 +00:00
@cli.argument ( ' -n ' , ' --dry-run ' , arg_only = True , action = ' store_true ' , help = " Don ' t write the data to disk. " )
2022-02-27 11:28:51 +00:00
@cli.argument ( ' -f ' , ' --filter ' , arg_only = True , action = ' append ' , default = [ ] , help = " Filter the list of keyboards based on partial name matches the supplied value. May be passed multiple times. " )
2022-08-06 06:14:29 +00:00
@cli.subcommand ( ' Generate QMK API data ' , hidden = False if cli . config . user . developer else True )
2020-10-25 21:48:44 +00:00
def generate_api ( cli ) :
""" Generates the QMK API data.
"""
2022-02-27 11:28:51 +00:00
v1_dir = BUILD_API_PATH / ' v1 '
2021-03-25 03:33:25 +00:00
keyboard_all_file = v1_dir / ' keyboards.json ' # A massive JSON containing everything
keyboard_list_file = v1_dir / ' keyboard_list.json ' # A simple list of keyboard targets
keyboard_aliases_file = v1_dir / ' keyboard_aliases.json ' # A list of historical keyboard names and their new name
2021-03-24 16:26:38 +00:00
keyboard_metadata_file = v1_dir / ' keyboard_metadata.json ' # All the data configurator/via needs for initialization
2022-11-23 18:01:07 +00:00
constants_metadata_file = v1_dir / ' constants_metadata.json ' # Metadata for available constants
2021-03-25 03:33:25 +00:00
usb_file = v1_dir / ' usb.json ' # A mapping of USB VID/PID -> keyboard target
2020-10-25 21:48:44 +00:00
2022-08-06 06:14:29 +00:00
if BUILD_API_PATH . exists ( ) :
shutil . rmtree ( BUILD_API_PATH )
shutil . copytree ( TEMPLATE_PATH , BUILD_API_PATH )
2022-11-08 03:03:02 +00:00
shutil . copytree ( DATA_PATH , v1_dir , copy_function = _filtered_copy )
2022-08-06 06:14:29 +00:00
2022-02-27 11:28:51 +00:00
# Filter down when required
2022-08-06 06:14:29 +00:00
keyboard_list = _filtered_keyboard_list ( )
2020-10-25 21:48:44 +00:00
2021-03-24 16:26:38 +00:00
kb_all = { }
usb_list = { }
2020-10-25 21:48:44 +00:00
# Generate and write keyboard specific JSON files
2022-02-27 11:28:51 +00:00
for keyboard_name in keyboard_list :
2023-01-20 03:38:19 +00:00
kb_json = info_json ( keyboard_name )
kb_all [ keyboard_name ] = kb_json
2023-01-11 01:38:35 +00:00
2020-10-25 21:48:44 +00:00
keyboard_dir = v1_dir / ' keyboards ' / keyboard_name
keyboard_info = keyboard_dir / ' info.json '
keyboard_readme = keyboard_dir / ' readme.md '
2021-05-25 02:38:27 +00:00
keyboard_readme_src = find_readme ( keyboard_name )
2020-10-25 21:48:44 +00:00
2023-01-20 03:38:19 +00:00
# Populate the list of JSON keymaps
for keymap in list_keymaps ( keyboard_name , c = False , fullpath = True ) :
2023-12-06 04:55:35 +00:00
keymap_rel = qmk . path . under_qmk_firmware ( keymap )
2023-01-20 03:38:19 +00:00
kb_json [ ' keymaps ' ] [ keymap . name ] = {
# TODO: deprecate 'url' as consumer needs to know its potentially hjson
2023-12-06 04:55:35 +00:00
' url ' : f ' https://raw.githubusercontent.com/qmk/qmk_firmware/master/ { keymap_rel } /keymap.json ' ,
2023-01-20 03:38:19 +00:00
# Instead consumer should grab from API and not repo directly
' path ' : ( keymap / ' keymap.json ' ) . as_posix ( ) ,
}
2020-10-25 21:48:44 +00:00
keyboard_dir . mkdir ( parents = True , exist_ok = True )
2023-03-29 12:01:16 +00:00
keyboard_json = json . dumps ( { ' last_updated ' : current_datetime ( ) , ' keyboards ' : { keyboard_name : kb_json } } , separators = ( ' , ' , ' : ' ) )
2021-05-10 18:18:44 +00:00
if not cli . args . dry_run :
2023-01-20 03:38:19 +00:00
keyboard_info . write_text ( keyboard_json , encoding = ' utf-8 ' )
2021-05-10 18:18:44 +00:00
cli . log . debug ( ' Wrote file %s ' , keyboard_info )
2020-10-25 21:48:44 +00:00
2021-05-25 02:38:27 +00:00
if keyboard_readme_src :
2022-02-27 11:28:51 +00:00
shutil . copyfile ( keyboard_readme_src , keyboard_readme )
2021-05-10 18:18:44 +00:00
cli . log . debug ( ' Copied %s -> %s ' , keyboard_readme_src , keyboard_readme )
2020-10-25 21:48:44 +00:00
2023-01-20 03:38:19 +00:00
# resolve keymaps as json
for keymap in kb_json [ ' keymaps ' ] :
keymap_hjson = kb_json [ ' keymaps ' ] [ keymap ] [ ' path ' ]
keymap_json = v1_dir / keymap_hjson
keymap_json . parent . mkdir ( parents = True , exist_ok = True )
2023-03-29 12:01:16 +00:00
keymap_json . write_text ( json . dumps ( json_load ( Path ( keymap_hjson ) ) , separators = ( ' , ' , ' : ' ) ) , encoding = ' utf-8 ' )
2023-01-20 03:38:19 +00:00
cli . log . debug ( ' Wrote keymap %s ' , keymap_json )
if ' usb ' in kb_json :
usb = kb_json [ ' usb ' ]
2020-10-25 21:48:44 +00:00
2021-03-24 16:26:38 +00:00
if ' vid ' in usb and usb [ ' vid ' ] not in usb_list :
usb_list [ usb [ ' vid ' ] ] = { }
2020-10-25 21:48:44 +00:00
2021-03-24 16:26:38 +00:00
if ' pid ' in usb and usb [ ' pid ' ] not in usb_list [ usb [ ' vid ' ] ] :
usb_list [ usb [ ' vid ' ] ] [ usb [ ' pid ' ] ] = { }
2020-10-25 21:48:44 +00:00
2020-12-30 18:27:37 +00:00
if ' vid ' in usb and ' pid ' in usb :
2021-03-24 16:26:38 +00:00
usb_list [ usb [ ' vid ' ] ] [ usb [ ' pid ' ] ] [ keyboard_name ] = usb
2020-10-25 21:48:44 +00:00
2021-05-10 18:18:44 +00:00
# Generate data for the global files
2021-03-24 16:26:38 +00:00
keyboard_list = sorted ( kb_all )
2023-11-22 00:14:34 +00:00
keyboard_aliases = keyboard_alias_definitions ( )
2021-03-24 16:26:38 +00:00
keyboard_metadata = {
' last_updated ' : current_datetime ( ) ,
' keyboards ' : keyboard_list ,
' keyboard_aliases ' : keyboard_aliases ,
2021-03-25 03:33:25 +00:00
' usb ' : usb_list ,
2021-03-24 16:26:38 +00:00
}
2021-05-10 18:18:44 +00:00
2022-11-05 10:30:09 +00:00
# Feature specific handling
_resolve_keycode_specs ( v1_dir )
2021-05-10 18:18:44 +00:00
# Write the global JSON files
2023-03-29 12:01:16 +00:00
keyboard_all_json = json . dumps ( { ' last_updated ' : current_datetime ( ) , ' keyboards ' : kb_all } , separators = ( ' , ' , ' : ' ) )
usb_json = json . dumps ( { ' last_updated ' : current_datetime ( ) , ' usb ' : usb_list } , separators = ( ' , ' , ' : ' ) )
keyboard_list_json = json . dumps ( { ' last_updated ' : current_datetime ( ) , ' keyboards ' : keyboard_list } , separators = ( ' , ' , ' : ' ) )
keyboard_aliases_json = json . dumps ( { ' last_updated ' : current_datetime ( ) , ' keyboard_aliases ' : keyboard_aliases } , separators = ( ' , ' , ' : ' ) )
keyboard_metadata_json = json . dumps ( keyboard_metadata , separators = ( ' , ' , ' : ' ) )
constants_metadata_json = json . dumps ( { ' last_updated ' : current_datetime ( ) , ' constants ' : _list_constants ( v1_dir ) } , separators = ( ' , ' , ' : ' ) )
2021-05-10 18:18:44 +00:00
if not cli . args . dry_run :
2023-01-20 03:38:19 +00:00
keyboard_all_file . write_text ( keyboard_all_json , encoding = ' utf-8 ' )
usb_file . write_text ( usb_json , encoding = ' utf-8 ' )
keyboard_list_file . write_text ( keyboard_list_json , encoding = ' utf-8 ' )
keyboard_aliases_file . write_text ( keyboard_aliases_json , encoding = ' utf-8 ' )
keyboard_metadata_file . write_text ( keyboard_metadata_json , encoding = ' utf-8 ' )
constants_metadata_file . write_text ( constants_metadata_json , encoding = ' utf-8 ' )