From 33f5e6740855066f3979ee923fa4ddb9ca706c5c Mon Sep 17 00:00:00 2001 From: Nick Brassel Date: Mon, 1 Jul 2024 21:49:21 +1000 Subject: [PATCH] Download relevant toolchain. --- .gitignore | 1 + lib/python/qmk/cli/__init__.py | 9 +++ lib/python/qmk/cli/toolchain/__init__.py | 0 lib/python/qmk/cli/toolchain/commands.py | 70 ++++++++++++++++++++++++ lib/python/qmk/constants.py | 6 ++ lib/python/qmk/util.py | 19 +++++++ requirements.txt | 5 +- 7 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 lib/python/qmk/cli/toolchain/__init__.py create mode 100644 lib/python/qmk/cli/toolchain/commands.py diff --git a/.gitignore b/.gitignore index 35b128606d7..10b70cccb0d 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ *.la *.stackdump *.sym +qmk_toolchains* # QMK-specific api_data/v1 diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py index b504aa5f8c6..0d3b47847b3 100644 --- a/lib/python/qmk/cli/__init__.py +++ b/lib/python/qmk/cli/__init__.py @@ -82,6 +82,7 @@ subcommands = [ 'qmk.cli.painter', 'qmk.cli.pytest', 'qmk.cli.test.c', + 'qmk.cli.toolchain.commands', 'qmk.cli.userspace.add', 'qmk.cli.userspace.compile', 'qmk.cli.userspace.doctor', @@ -258,3 +259,11 @@ for subcommand in subcommands: _eprint(f'Warning: Could not import {subcommand}: {e.__class__.__name__}, {e}') else: raise + +# Add the QMK toolchains to $PATH +try: + from qmk.constants import QMK_TOOLCHAINS_PATH # noqa + if QMK_TOOLCHAINS_PATH.exists(): + os.environ['PATH'] = f'{QMK_TOOLCHAINS_PATH}/bin{os.pathsep}{os.environ["PATH"]}' +except Exception: + pass diff --git a/lib/python/qmk/cli/toolchain/__init__.py b/lib/python/qmk/cli/toolchain/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/lib/python/qmk/cli/toolchain/commands.py b/lib/python/qmk/cli/toolchain/commands.py new file mode 100644 index 00000000000..b9a6d48d67d --- /dev/null +++ b/lib/python/qmk/cli/toolchain/commands.py @@ -0,0 +1,70 @@ +"""This script sets up the QMK-provided toolchains. +""" +import platform +import shutil +import subprocess +import os +from milc import cli +from pathlib import Path + +from qmk.constants import QMK_TOOLCHAINS_PATH, QMK_TOOLCHAINS_TAG +from qmk.util import download_with_progress, cached_get + + +def _os_prefix(): + os = platform.system().lower() + if "linux" in os: + return "linux" + elif "darwin" in os or "mac" in os or "macos" in os or "osx" in os or "macosx" in os: + return "macos" + elif "windows" in os or "cygwin" in os or "msys" in os or "win32" in os or "win64" in os or "nt" in os: + return "windows" + return "unknown" + + +def _arch_suffix(): + arch = platform.machine().lower() + if "amd64" in arch or "x86_64" in arch or "x64" in arch or "x86-64" in arch: + return "X64" + elif "arm64" in arch or "aarch64" in arch: + return "ARM64" + return "unknown" + + +def _os_arch(): + return f'{_os_prefix()}{_arch_suffix()}' + + +def _gcc_version_str(gcc_exe): + lines = cli.run([gcc_exe, '-v'], check=True, capture_output=True, combined_output=True, stdin=subprocess.DEVNULL).stdout.strip().splitlines() + return list(filter(lambda x: x.startswith('gcc version'), lines))[0].strip() + + +@cli.argument('-t', '--tag', default=QMK_TOOLCHAINS_TAG, help=f'The tag of the QMK-provided toolchains to download. Defaults to `{QMK_TOOLCHAINS_TAG}`.') +@cli.subcommand('Set up the QMK-provided toolchains.', hidden=False if cli.config.user.developer else True) +def toolchain_setup(cli): + """Set up the QMK-provided toolchains. + """ + + # Remove the toolchains directory if it exists already + if QMK_TOOLCHAINS_PATH.exists(): + cli.log.info(f'Removing existing toolchains at `{QMK_TOOLCHAINS_PATH}`...') + shutil.rmtree(QMK_TOOLCHAINS_PATH) + + url = f'https://api.github.com/repos/qmk/qmk_toolchains/releases/tags/{cli.args.tag}' + payload = cached_get(url) + if payload.status_code != 200: + cli.log.error(f'Failed to fetch toolchain metadata from `{url}`.') + return + downloadables = filter(lambda x: _os_arch() in x['name'], payload.json()['assets']) + for downloadable in downloadables: + if not Path(downloadable["name"]).exists(): + download_with_progress(downloadable['browser_download_url'], downloadable["name"]) + cli.log.info(f'Extracting `{downloadable["name"]}` to `{QMK_TOOLCHAINS_PATH}`...') + QMK_TOOLCHAINS_PATH.mkdir(parents=True, exist_ok=True) + cli.run(['tar', 'xf', downloadable["name"], '-C', QMK_TOOLCHAINS_PATH, '--strip-components=1'], capture_output=False, check=True, stdin=subprocess.DEVNULL) + + os.environ['PATH'] = f'{QMK_TOOLCHAINS_PATH}/bin{os.pathsep}{os.environ["PATH"]}' + + for gcc_exe in ['avr-gcc', 'arm-none-eabi-gcc', 'riscv32-unknown-elf-gcc']: + cli.log.info(f'{gcc_exe:24s}: {_gcc_version_str(gcc_exe)}') diff --git a/lib/python/qmk/constants.py b/lib/python/qmk/constants.py index 90e4452f2b9..622b2b9ab0e 100644 --- a/lib/python/qmk/constants.py +++ b/lib/python/qmk/constants.py @@ -21,6 +21,12 @@ QMK_FIRMWARE_UPSTREAM = 'qmk/qmk_firmware' # This is the number of directories under `qmk_firmware/keyboards` that will be traversed. This is currently a limitation of our make system. MAX_KEYBOARD_SUBFOLDERS = 5 +# Intended qmk/qmk_toolchains release to deploy +QMK_TOOLCHAINS_TAG = 'v14.1.0-4' + +# QMK Toolchains path +QMK_TOOLCHAINS_PATH = Path('~/.local/qmk/toolchains').expanduser() + # Supported processor types CHIBIOS_PROCESSORS = 'cortex-m0', 'cortex-m0plus', 'cortex-m3', 'cortex-m4', 'MKL26Z64', 'MK20DX128', 'MK20DX256', 'MK64FX512', 'MK66FX1M0', 'RP2040', 'STM32F042', 'STM32F072', 'STM32F103', 'STM32F303', 'STM32F401', 'STM32F405', 'STM32F407', 'STM32F411', 'STM32F446', 'STM32G431', 'STM32G474', 'STM32H723', 'STM32H733', 'STM32L412', 'STM32L422', 'STM32L432', 'STM32L433', 'STM32L442', 'STM32L443', 'GD32VF103', 'WB32F3G71', 'WB32FQ95' LUFA_PROCESSORS = 'at90usb162', 'atmega16u2', 'atmega32u2', 'atmega16u4', 'atmega32u4', 'at90usb646', 'at90usb647', 'at90usb1286', 'at90usb1287', None diff --git a/lib/python/qmk/util.py b/lib/python/qmk/util.py index b73fab89d12..125bc768c57 100644 --- a/lib/python/qmk/util.py +++ b/lib/python/qmk/util.py @@ -3,6 +3,7 @@ import contextlib import multiprocessing import sys +from pathlib import Path from milc import cli @@ -27,6 +28,24 @@ def maybe_exit_config(should_exit: bool = True, should_reraise: bool = False): maybe_exit_reraise = should_reraise +def cached_get(*args, **kwargs): + import requests_cache + session = requests_cache.CachedSession(Path('~/.local/qmk/qmk_requests.sqlite').expanduser(), expire_after=300, cache_control=True) + return session.get(*args, **kwargs) + + +def download_with_progress(url, filename): + import requests + import tqdm + response = requests.get(url, stream=True) + total_size = int(response.headers.get('content-length', 0)) + with tqdm.tqdm(desc=filename, total=total_size, unit='B', unit_scale=True) as pbar: + with open(filename, 'wb') as file: + for data in response.iter_content(1024): + file.write(data) + pbar.update(len(data)) + + @contextlib.contextmanager def parallelize(): """Returns a function that can be used in place of a map() call. diff --git a/requirements.txt b/requirements.txt index 6bee7463243..1738c7a61e2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,8 +6,11 @@ dotty-dict hid hjson jsonschema>=4 -milc>=1.4.2 +milc>=1.8.0 pygments pyserial pyusb pillow +requests +requests-cache +tqdm