From 575b43e67dec85cdf58025806f3c993f80f70705 Mon Sep 17 00:00:00 2001 From: hathach Date: Sat, 23 Jan 2021 17:32:24 +0700 Subject: [PATCH] seperate ci build for family and orphaned boards --- .github/workflows/build.yml | 69 ++++++++- examples/device/uac2_headset/.skip.MCU_SAMD11 | 0 examples/device/uac2_headset/.skip.MCU_SAME5X | 0 examples/device/uac2_headset/.skip.MCU_SAMG | 0 tools/{build_all.py => build_board.py} | 100 ++++++------- tools/build_family.py | 136 ++++++++++++++++++ 6 files changed, 241 insertions(+), 64 deletions(-) create mode 100644 examples/device/uac2_headset/.skip.MCU_SAMD11 create mode 100644 examples/device/uac2_headset/.skip.MCU_SAME5X create mode 100644 examples/device/uac2_headset/.skip.MCU_SAMG rename tools/{build_all.py => build_board.py} (56%) create mode 100644 tools/build_family.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4f18790a0..de1b9bafd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,8 +20,8 @@ jobs: cd test ceedling test:all - # Build most of the ports - build: + # Build all no-family (opharned) boards + build-board: runs-on: ubuntu-latest strategy: fail-fast: false @@ -40,6 +40,7 @@ jobs: - 'device/midi_test' - 'device/msc_dual_lun' - 'device/net_lwip_webserver' + - 'device/uac2_headset' - 'device/usbtmc' - 'device/webusb_serial' - 'host/cdc_msc_hid' @@ -86,11 +87,71 @@ jobs: run: | # some submodule has it own submodule that need to be fetched as well git submodule update --init --recursive hw/mcu/microchip - #git submodule update --init --recursive hw/mcu/microchip + git submodule update --init --recursive lib/FreeRTOS - name: Build + run: python3 tools/build_board.py ${{ matrix.example }} + + # build all example for each family + build-family: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + family: + - 'imxrt' + - 'nrf' + #- 'rp2040' + - 'samd21' + - 'samd51' + - 'stm32f4' + - 'stm32f7' + steps: + - name: Setup Python + uses: actions/setup-python@v1 + + - name: Setup Node.js + uses: actions/setup-node@v1 + + - name: Cache MSP430 Toolchain + id: cache-msp430 + uses: actions/cache@v1 + with: + path: /tmp/dl/ + # Increment gcc version number at end when updating downloads + key: msp430-${{ runner.os }}-9.2.0.50 + + - name: Install Toolchains + env: + MSP430GCC_URL: http://software-dl.ti.com/msp430/msp430_public_sw/mcu/msp430/MSPGCC/9_2_0_0/export/msp430-gcc-9.2.0.50_linux64.tar.bz2 run: | - python3 tools/build_all.py ${{ matrix.example }} + # ARM & RISC-V GCC from xpack + npm install --global xpm + xpm install --global @xpack-dev-tools/arm-none-eabi-gcc@latest + xpm install --global @xpack-dev-tools/riscv-none-embed-gcc@latest + echo `echo $HOME/opt/xPacks/@xpack-dev-tools/arm-none-eabi-gcc/*/.content/bin` >> $GITHUB_PATH + echo `echo $HOME/opt/xPacks/@xpack-dev-tools/riscv-none-embed-gcc/*/.content/bin` >> $GITHUB_PATH + + # TI MSP430 GCC + mkdir -p /tmp/dl/ + [ -f "/tmp/dl/msp430-gcc.tar.bz2" ] || wget --progress=dot:mega $MSP430GCC_URL -O /tmp/dl/msp430-gcc.tar.bz2 + tar -C $HOME -xaf /tmp/dl/msp430-gcc.tar.bz2 + echo `echo $HOME/msp430-gcc-*_linux64/bin` >> $GITHUB_PATH + + - name: Checkout TinyUSB + uses: actions/checkout@v2 + with: + # Cannot do submodule checkout here since LWIP's git server cannot checkout unadventised commits (it must use tags) + submodules: 'true' + + - name: Checkout Sub-Submodules + run: | + # some submodule has it own submodule that need to be fetched as well + git submodule update --init --recursive hw/mcu/microchip + git submodule update --init --recursive lib/FreeRTOS + + - name: Build + run: python3 tools/build_family.py ${{ matrix.family }} # Build ESP32S build-esp32s: diff --git a/examples/device/uac2_headset/.skip.MCU_SAMD11 b/examples/device/uac2_headset/.skip.MCU_SAMD11 new file mode 100644 index 000000000..e69de29bb diff --git a/examples/device/uac2_headset/.skip.MCU_SAME5X b/examples/device/uac2_headset/.skip.MCU_SAME5X new file mode 100644 index 000000000..e69de29bb diff --git a/examples/device/uac2_headset/.skip.MCU_SAMG b/examples/device/uac2_headset/.skip.MCU_SAMG new file mode 100644 index 000000000..e69de29bb diff --git a/tools/build_all.py b/tools/build_board.py similarity index 56% rename from tools/build_all.py rename to tools/build_board.py index 787011e26..cf3a8fb94 100644 --- a/tools/build_all.py +++ b/tools/build_board.py @@ -18,48 +18,60 @@ total_time = time.monotonic() build_format = '| {:29} | {:30} | {:18} | {:7} | {:6} | {:6} |' build_separator = '-' * 106 +def filter_with_input(mylist): + if len(sys.argv) > 1: + input_args = list(set(mylist).intersection(sys.argv)) + if len(input_args) > 0: + mylist[:] = input_args + # If examples are not specified in arguments, build all all_examples = [] - for entry in os.scandir("examples/device"): if entry.is_dir(): all_examples.append("device/" + entry.name) - for entry in os.scandir("examples/host"): if entry.is_dir(): all_examples.append("host/" + entry.name) - -if len(sys.argv) > 1: - input_examples = list(set(all_examples).intersection(sys.argv)) - if len(input_examples) > 0: - all_examples = input_examples - +filter_with_input(all_examples) all_examples.sort() # If boards are not specified in arguments, build all all_boards = [] - for entry in os.scandir("hw/bsp"): - if entry.is_dir() and entry.name != "esp32s2": - if os.path.isdir(entry.path + "/boards"): - # family directory - for subentry in os.scandir(entry.path + "/boards"): - if subentry.is_dir(): all_boards.append(subentry.name) - else: - all_boards.append(entry.name) - -if len(sys.argv) > 1: - input_boards = list(set(all_boards).intersection(sys.argv)) - if len(input_boards) > 0: - all_boards = input_boards - + if entry.is_dir() and os.path.exists(entry.path + "/board.mk"): + all_boards.append(entry.name) +filter_with_input(all_boards) all_boards.sort() -def build_example(example, board): - subprocess.run("make -C examples/{} BOARD={} clean".format(example, board), shell=True, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - return subprocess.run("make -j -C examples/{} BOARD={} all".format(example, board), shell=True, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) +def build_board(example, board): + global success_count, fail_count, skip_count + start_time = time.monotonic() + flash_size = "-" + sram_size = "-" + + # Check if board is skipped + if skip_example(example, board): + success = SKIPPED + skip_count += 1 + print(build_format.format(example, board, success, '-', flash_size, sram_size)) + else: + subprocess.run("make -C examples/{} BOARD={} clean".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + build_result = subprocess.run("make -j -C examples/{} BOARD={} all".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + + if build_result.returncode == 0: + success = SUCCEEDED + success_count += 1 + (flash_size, sram_size) = build_size(example, board) + else: + exit_status = build_result.returncode + success = FAILED + fail_count += 1 + + build_duration = time.monotonic() - start_time + print(build_format.format(example, board, success, "{:.2f}s".format(build_duration), flash_size, sram_size)) + + if build_result.returncode != 0: + print(build_result.stdout.decode("utf-8")) def build_size(example, board): #elf_file = 'examples/device/{}/_build/build-{}/{}-firmware.elf'.format(example, board, board) @@ -72,11 +84,7 @@ def build_size(example, board): def skip_example(example, board): ex_dir = 'examples/' + example - board_mk = 'hw/bsp/{}/board.mk'.format(board) - if not os.path.exists(board_mk): - board_mk = list(glob.iglob('hw/bsp/*/boards/{}/../../family.mk'.format(board)))[0] - with open(board_mk) as mk: mk_contents = mk.read() @@ -107,35 +115,7 @@ print(build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', ' for example in all_examples: print(build_separator) for board in all_boards: - start_time = time.monotonic() - - flash_size = "-" - sram_size = "-" - - # Check if board is skipped - if skip_example(example, board): - success = SKIPPED - skip_count += 1 - print(build_format.format(example, board, success, '-', flash_size, sram_size)) - else: - build_result = build_example(example, board) - - if build_result.returncode == 0: - success = SUCCEEDED - success_count += 1 - (flash_size, sram_size) = build_size(example, board) - else: - exit_status = build_result.returncode - success = FAILED - fail_count += 1 - - build_duration = time.monotonic() - start_time - print(build_format.format(example, board, success, "{:.2f}s".format(build_duration), flash_size, sram_size)) - - if build_result.returncode != 0: - print(build_result.stdout.decode("utf-8")) - - + build_board(example, board) total_time = time.monotonic() - total_time print(build_separator) diff --git a/tools/build_family.py b/tools/build_family.py new file mode 100644 index 000000000..7f2c1a706 --- /dev/null +++ b/tools/build_family.py @@ -0,0 +1,136 @@ +import os +import glob +import sys +import subprocess +import time + +SUCCEEDED = "\033[32msucceeded\033[0m" +FAILED = "\033[31mfailed\033[0m" +SKIPPED = "\033[33mskipped\033[0m" + +success_count = 0 +fail_count = 0 +skip_count = 0 +exit_status = 0 + +total_time = time.monotonic() + +build_format = '| {:29} | {:30} | {:18} | {:7} | {:6} | {:6} |' +build_separator = '-' * 106 + +def filter_with_input(mylist): + if len(sys.argv) > 1: + input_args = list(set(mylist).intersection(sys.argv)) + if len(input_args) > 0: + mylist[:] = input_args + +# If examples are not specified in arguments, build all +all_examples = [] +for entry in os.scandir("examples/device"): + if entry.is_dir(): + all_examples.append("device/" + entry.name) +for entry in os.scandir("examples/host"): + if entry.is_dir(): + all_examples.append("host/" + entry.name) +filter_with_input(all_examples) +all_examples.sort() + +# If family are not specified in arguments, build all +all_families = [] +for entry in os.scandir("hw/bsp"): + if entry.is_dir() and os.path.isdir(entry.path + "/boards") and entry.name != "esp32s2": + all_families.append(entry.name) + +filter_with_input(all_families) +all_families.sort() + +def build_family(example, family): + all_boards = [] + for entry in os.scandir("hw/bsp/{}/boards".format(family)): + if entry.is_dir(): + all_boards.append(entry.name) + all_boards.sort() + + for board in all_boards: + build_board(example, board) + +def build_board(example, board): + global success_count, fail_count, skip_count + start_time = time.monotonic() + flash_size = "-" + sram_size = "-" + + # Check if board is skipped + if skip_example(example, board): + success = SKIPPED + skip_count += 1 + print(build_format.format(example, board, success, '-', flash_size, sram_size)) + else: + subprocess.run("make -C examples/{} BOARD={} clean".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + build_result = subprocess.run("make -j -C examples/{} BOARD={} all".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + + if build_result.returncode == 0: + success = SUCCEEDED + success_count += 1 + (flash_size, sram_size) = build_size(example, board) + else: + exit_status = build_result.returncode + success = FAILED + fail_count += 1 + + build_duration = time.monotonic() - start_time + print(build_format.format(example, board, success, "{:.2f}s".format(build_duration), flash_size, sram_size)) + + if build_result.returncode != 0: + print(build_result.stdout.decode("utf-8")) + +def build_size(example, board): + #elf_file = 'examples/device/{}/_build/build-{}/{}-firmware.elf'.format(example, board, board) + elf_file = 'examples/{}/_build/build-{}/*.elf'.format(example, board) + size_output = subprocess.run('size {}'.format(elf_file), shell=True, stdout=subprocess.PIPE).stdout.decode("utf-8") + size_list = size_output.split('\n')[1].split('\t') + flash_size = int(size_list[0]) + sram_size = int(size_list[1]) + int(size_list[2]) + return (flash_size, sram_size) + +def skip_example(example, board): + ex_dir = 'examples/' + example + board_mk = 'hw/bsp/{}/family.mk'.format(family) + + with open(board_mk) as mk: + mk_contents = mk.read() + + # Skip all OPT_MCU_NONE these are WIP port + if '-DCFG_TUSB_MCU=OPT_MCU_NONE' in mk_contents: + return 1 + + # Skip if CFG_TUSB_MCU in board.mk to match skip file + for skip_file in glob.iglob(ex_dir + '/.skip.MCU_*'): + mcu_cflag = '-DCFG_TUSB_MCU=OPT_' + os.path.basename(skip_file).split('.')[2] + if mcu_cflag in mk_contents: + return 1 + + # Build only list, if exists only these MCU are built + only_list = list(glob.iglob(ex_dir + '/.only.MCU_*')) + if len(only_list) > 0: + for only_file in only_list: + mcu_cflag = '-DCFG_TUSB_MCU=OPT_' + os.path.basename(only_file).split('.')[2] + if mcu_cflag in mk_contents: + return 0 + return 1 + return 0 + +print(build_separator) +print(build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM')) + +for example in all_examples: + print(build_separator) + for family in all_families: + build_family(example, family) + +total_time = time.monotonic() - total_time +print(build_separator) +print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(success_count, SUCCEEDED, fail_count, FAILED, skip_count, SKIPPED, total_time)) +print(build_separator) + +sys.exit(exit_status)