Merge pull request #1544 from hathach/ci-parallel-build

Ci parallel build
This commit is contained in:
Ha Thach 2022-07-01 20:52:46 +07:00 committed by GitHub
commit ecb899408b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 219 additions and 183 deletions

View File

@ -55,6 +55,9 @@ jobs:
- name: Set Toolchain Path
run: echo >> $GITHUB_PATH `echo ~/cache/toolchain/*/bin`
- name: Get Dependencies
run: python3 tools/get_dependencies.py ${{ matrix.family }}
- name: Build
run: python3 tools/build_family.py ${{ matrix.family }}

View File

@ -67,6 +67,11 @@ jobs:
- name: Setup Python
uses: actions/setup-python@v3
- name: Install ARM GCC
uses: carlosperate/arm-none-eabi-gcc-action@v1
with:
release: '11.2-2022.02'
- name: Checkout TinyUSB
uses: actions/checkout@v3
@ -84,12 +89,9 @@ jobs:
run: |
git clone --depth 1 -b develop https://github.com/raspberrypi/pico-sdk ~/pico-sdk
echo >> $GITHUB_ENV PICO_SDK_PATH=~/pico-sdk
git submodule update --init hw/mcu/raspberry_pi/Pico-PIO-USB
- name: Install ARM GCC
uses: carlosperate/arm-none-eabi-gcc-action@v1
with:
release: '11.2-2022.02'
- name: Get Dependencies
run: python3 tools/get_dependencies.py ${{ matrix.family }}
- name: Build
run: python3 tools/build_family.py ${{ matrix.family }}
@ -122,16 +124,16 @@ jobs:
- name: Setup Python
uses: actions/setup-python@v3
- name: Install ARM GCC
uses: carlosperate/arm-none-eabi-gcc-action@v1
with:
release: '11.2-2022.02'
- name: Checkout TinyUSB
uses: actions/checkout@v3
- name: Checkout common submodules in lib
run: git submodule update --init lib/FreeRTOS-Kernel lib/lwip
- name: Install ARM GCC
uses: carlosperate/arm-none-eabi-gcc-action@v1
with:
release: '11.2-2022.02'
- name: Build
run: python3 tools/build_board.py ${{ matrix.example }}

View File

@ -52,6 +52,9 @@ jobs:
- name: Set Toolchain Path
run: echo >> $GITHUB_PATH `echo ~/cache/toolchain/*/bin`
- name: Get Dependencies
run: python3 tools/get_dependencies.py ${{ matrix.family }}
- name: Build
run: python3 tools/build_family.py ${{ matrix.family }}

View File

@ -53,6 +53,9 @@ jobs:
- name: Set Toolchain Path
run: echo >> $GITHUB_PATH `echo ~/cache/toolchain/*/bin`
- name: Get Dependencies
run: python3 tools/get_dependencies.py ${{ matrix.family }}
- name: Build
run: python3 tools/build_family.py ${{ matrix.family }}

View File

@ -53,6 +53,9 @@ jobs:
- name: Set Toolchain Path
run: echo >> $GITHUB_PATH `echo ~/cache/toolchain/*/bin`
- name: Get Dependencies
run: python3 tools/get_dependencies.py ${{ matrix.family }}
- name: Build
run: python3 tools/build_family.py ${{ matrix.family }}

View File

@ -50,7 +50,12 @@ Some TinyUSB examples also requires external submodule libraries in ``/lib`` suc
$ git submodule update --init lib
In addition, MCU driver submodule is also needed to provide low-level MCU peripheral's driver. Luckily, it will be fetched if needed when you run the ``make`` to build your board.
In addition, MCU driver submodule is also needed to provide low-level MCU peripheral's driver. To download these depencies for your board, run the ``get-dpes`` as follow.
.. code-block::
$ make BOARD=feather_nrf52840_express get-deps
Some modules will also require a module-specific SDK (e.g. RP2040) or binary (e.g. Sony Spresense) to build examples.

View File

@ -45,11 +45,6 @@ else
SRC_C += $(subst $(TOP)/,,$(wildcard $(TOP)/$(FAMILY_PATH)/*.c))
endif
# Fetch submodules depended by family
fetch_submodule_if_empty = $(if $(wildcard $(TOP)/$1/*),,$(info $(shell git -C $(TOP) submodule update --init $1)))
ifdef DEPS_SUBMODULES
$(foreach s,$(DEPS_SUBMODULES),$(call fetch_submodule_if_empty,$(s)))
endif
#-------------- Cross Compiler ------------
# Can be set by board, default to ARM GCC

View File

@ -5,6 +5,7 @@
# Set all as default goal
.DEFAULT_GOAL := all
# ---------------- GNU Make Start -----------------------
# ESP32-Sx and RP2040 has its own CMake build system
ifeq (,$(findstring $(FAMILY),esp32s2 esp32s3 rp2040))
@ -141,7 +142,23 @@ $(BUILD)/obj/%_asm.o: %.S
@echo AS $(notdir $@)
@$(CC) -x assembler-with-cpp $(ASFLAGS) -c -o $@ $<
endif # GNU Make
endif
.PHONY: clean
clean:
ifeq ($(CMDEXE),1)
rd /S /Q $(subst /,\,$(BUILD))
else
$(RM) -rf $(BUILD)
endif
# ---------------- GNU Make End -----------------------
# get depenecies
.PHONY: get-deps
get-deps:
ifdef DEPS_SUBMODULES
git -C $(TOP) submodule update --init $(DEPS_SUBMODULES)
endif
size: $(BUILD)/$(PROJECT).elf
-@echo ''
@ -152,14 +169,6 @@ size: $(BUILD)/$(PROJECT).elf
linkermap: $(BUILD)/$(PROJECT).elf
@linkermap -v $<.map
.PHONY: clean
clean:
ifeq ($(CMDEXE),1)
rd /S /Q $(subst /,\,$(BUILD))
else
$(RM) -rf $(BUILD)
endif
# ---------------------------------------
# Flash Targets
# ---------------------------------------

View File

@ -1,6 +1,8 @@
JLINK_DEVICE = rp2040_m0_0
PYOCD_TARGET = rp2040
DEPS_SUBMODULES += hw/mcu/raspberry_pi/Pico-PIO-USB
ifeq ($(DEBUG), 1)
CMAKE_DEFSYM += -DCMAKE_BUILD_TYPE=Debug
endif

View File

@ -1,8 +1,8 @@
import os
import glob
import sys
import subprocess
import time
import subprocess
from multiprocessing import Pool
import build_utils
@ -10,90 +10,60 @@ SUCCEEDED = "\033[32msucceeded\033[0m"
FAILED = "\033[31mfailed\033[0m"
SKIPPED = "\033[33mskipped\033[0m"
success_count = 0
fail_count = 0
skip_count = 0
exit_status = 0
total_time = time.monotonic()
build_format = '| {:29} | {:30} | {:18} | {:7} | {:6} | {:6} |'
build_separator = '-' * 106
def filter_with_input(mylist):
if len(sys.argv) > 1:
input_args = list(set(mylist).intersection(sys.argv))
if len(input_args) > 0:
mylist[:] = input_args
# If examples are not specified in arguments, build all
all_examples = []
for dir1 in os.scandir("examples"):
if dir1.is_dir():
for entry in os.scandir(dir1.path):
if entry.is_dir():
all_examples.append(dir1.name + '/' + entry.name)
filter_with_input(all_examples)
all_examples.sort()
# If boards are not specified in arguments, build all
all_boards = []
for entry in os.scandir("hw/bsp"):
if entry.is_dir() and os.path.exists(entry.path + "/board.mk"):
all_boards.append(entry.name)
filter_with_input(all_boards)
all_boards.sort()
if __name__ == '__main__':
# If examples are not specified in arguments, build all
all_examples = []
for dir1 in os.scandir("examples"):
if dir1.is_dir():
for entry in os.scandir(dir1.path):
if entry.is_dir():
all_examples.append(dir1.name + '/' + entry.name)
filter_with_input(all_examples)
all_examples.sort()
def build_board(example, board):
global success_count, fail_count, skip_count, exit_status
start_time = time.monotonic()
flash_size = "-"
sram_size = "-"
# If boards are not specified in arguments, build all
all_boards = []
for entry in os.scandir("hw/bsp"):
if entry.is_dir() and os.path.exists(entry.path + "/board.mk"):
all_boards.append(entry.name)
filter_with_input(all_boards)
all_boards.sort()
# Check if board is skipped
if build_utils.skip_example(example, board):
success = SKIPPED
skip_count += 1
print(build_format.format(example, board, success, '-', flash_size, sram_size))
else:
subprocess.run("make -C examples/{} BOARD={} clean".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
build_result = subprocess.run("make -j -C examples/{} BOARD={} all".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
# Get dependencies
for b in all_boards:
subprocess.run("make -C examples/device/board_test BOARD={} get-deps".format(b), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
if build_result.returncode == 0:
success = SUCCEEDED
success_count += 1
(flash_size, sram_size) = build_size(example, board)
else:
exit_status = build_result.returncode
success = FAILED
fail_count += 1
build_duration = time.monotonic() - start_time
print(build_format.format(example, board, success, "{:.2f}s".format(build_duration), flash_size, sram_size))
if build_result.returncode != 0:
print(build_result.stdout.decode("utf-8"))
def build_size(example, board):
#elf_file = 'examples/device/{}/_build/{}/{}-firmware.elf'.format(example, board, board)
elf_file = 'examples/{}/_build/{}/*.elf'.format(example, board)
size_output = subprocess.run('size {}'.format(elf_file), shell=True, stdout=subprocess.PIPE).stdout.decode("utf-8")
size_list = size_output.split('\n')[1].split('\t')
flash_size = int(size_list[0])
sram_size = int(size_list[1]) + int(size_list[2])
return (flash_size, sram_size)
print(build_separator)
print(build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM'))
for example in all_examples:
print(build_separator)
for board in all_boards:
build_board(example, board)
print(build_utils.build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM'))
total_time = time.monotonic()
total_time = time.monotonic() - total_time
print(build_separator)
print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(success_count, SUCCEEDED, fail_count, FAILED, skip_count, SKIPPED, total_time))
print(build_separator)
# succeeded, failed, skipped
total_result = [0, 0, 0]
for example in all_examples:
print(build_separator)
with Pool(processes=os.cpu_count()) as pool:
pool_args = list((map(lambda b, e=example: [e, b], all_boards)))
result = pool.starmap(build_utils.build_example, pool_args)
# sum all element of same index (column sum)
result = list(map(sum, list(zip(*result))))
# add to total result
total_result = list(map(lambda x, y: x + y, total_result, result))
sys.exit(exit_status)
total_time = time.monotonic() - total_time
print(build_separator)
print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(total_result[0], SUCCEEDED, total_result[1],
FAILED, total_result[2], SKIPPED, total_time))
print(build_separator)
sys.exit(total_result[1])

View File

@ -1,8 +1,7 @@
import os
import glob
import sys
import subprocess
import time
from multiprocessing import Pool
import build_utils
@ -10,40 +9,15 @@ SUCCEEDED = "\033[32msucceeded\033[0m"
FAILED = "\033[31mfailed\033[0m"
SKIPPED = "\033[33mskipped\033[0m"
success_count = 0
fail_count = 0
skip_count = 0
exit_status = 0
total_time = time.monotonic()
build_format = '| {:29} | {:30} | {:18} | {:7} | {:6} | {:6} |'
build_separator = '-' * 106
def filter_with_input(mylist):
if len(sys.argv) > 1:
input_args = list(set(mylist).intersection(sys.argv))
if len(input_args) > 0:
mylist[:] = input_args
# If examples are not specified in arguments, build all
all_examples = []
for dir1 in os.scandir("examples"):
if dir1.is_dir():
for entry in os.scandir(dir1.path):
if entry.is_dir():
all_examples.append(dir1.name + '/' + entry.name)
filter_with_input(all_examples)
all_examples.sort()
# If family are not specified in arguments, build all
all_families = []
for entry in os.scandir("hw/bsp"):
if entry.is_dir() and os.path.isdir(entry.path + "/boards") and entry.name not in ("esp32s2", "esp32s3"):
all_families.append(entry.name)
filter_with_input(all_families)
all_families.sort()
def build_family(example, family):
all_boards = []
@ -52,61 +26,49 @@ def build_family(example, family):
all_boards.append(entry.name)
filter_with_input(all_boards)
all_boards.sort()
for board in all_boards:
build_board(example, board)
def build_board(example, board):
global success_count, fail_count, skip_count, exit_status
start_time = time.monotonic()
flash_size = "-"
sram_size = "-"
# Check if board is skipped
if build_utils.skip_example(example, board):
success = SKIPPED
skip_count += 1
print(build_format.format(example, board, success, '-', flash_size, sram_size))
else:
#subprocess.run("make -C examples/{} BOARD={} clean".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
build_result = subprocess.run("make -j -C examples/{} BOARD={} all".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
with Pool(processes=os.cpu_count()) as pool:
pool_args = list((map(lambda b, e=example: [e, b], all_boards)))
result = pool.starmap(build_utils.build_example, pool_args)
# sum all element of same index (column sum)
return list(map(sum, list(zip(*result))))
if build_result.returncode == 0:
success = SUCCEEDED
success_count += 1
(flash_size, sram_size) = build_size(example, board)
subprocess.run("make -j -C examples/{} BOARD={} copy-artifact".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
else:
exit_status = build_result.returncode
success = FAILED
fail_count += 1
build_duration = time.monotonic() - start_time
print(build_format.format(example, board, success, "{:.2f}s".format(build_duration), flash_size, sram_size))
if __name__ == '__main__':
# If examples are not specified in arguments, build all
all_examples = []
for dir1 in os.scandir("examples"):
if dir1.is_dir():
for entry in os.scandir(dir1.path):
if entry.is_dir():
all_examples.append(dir1.name + '/' + entry.name)
filter_with_input(all_examples)
all_examples.sort()
if build_result.returncode != 0:
print(build_result.stdout.decode("utf-8"))
# If family are not specified in arguments, build all
all_families = []
for entry in os.scandir("hw/bsp"):
if entry.is_dir() and os.path.isdir(entry.path + "/boards") and entry.name not in ("esp32s2", "esp32s3"):
all_families.append(entry.name)
filter_with_input(all_families)
all_families.sort()
def build_size(example, board):
#elf_file = 'examples/device/{}/_build/{}/{}-firmware.elf'.format(example, board, board)
elf_file = 'examples/{}/_build/{}/*.elf'.format(example, board)
size_output = subprocess.run('size {}'.format(elf_file), shell=True, stdout=subprocess.PIPE).stdout.decode("utf-8")
size_list = size_output.split('\n')[1].split('\t')
flash_size = int(size_list[0])
sram_size = int(size_list[1]) + int(size_list[2])
return (flash_size, sram_size)
print(build_separator)
print(build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM'))
for example in all_examples:
print(build_separator)
for family in all_families:
build_family(example, family)
print(build_utils.build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM'))
total_time = time.monotonic()
total_time = time.monotonic() - total_time
print(build_separator)
print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(success_count, SUCCEEDED, fail_count, FAILED, skip_count, SKIPPED, total_time))
print(build_separator)
# succeeded, failed, skipped
total_result = [0, 0, 0]
for example in all_examples:
print(build_separator)
for family in all_families:
fret = build_family(example, family)
total_result = list(map(lambda x, y: x + y, total_result, fret))
sys.exit(exit_status)
total_time = time.monotonic() - total_time
print(build_separator)
print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(total_result[0], SUCCEEDED, total_result[1],
FAILED, total_result[2], SKIPPED, total_time))
print(build_separator)
sys.exit(total_result[1])

View File

@ -1,9 +1,18 @@
import subprocess
import pathlib
import time
build_format = '| {:29} | {:30} | {:18} | {:7} | {:6} | {:6} |'
SUCCEEDED = "\033[32msucceeded\033[0m"
FAILED = "\033[31mfailed\033[0m"
SKIPPED = "\033[33mskipped\033[0m"
def skip_example(example, board):
ex_dir = pathlib.Path('examples/') / example
bsp = pathlib.Path("hw/bsp")
if (bsp / board / "board.mk").exists():
# board without family
board_dir = bsp / board
@ -15,19 +24,19 @@ def skip_example(example, board):
if not board_dir:
# Skip unknown boards
return True
board_dir = list(board_dir)[0]
family_dir = board_dir.parent.parent
family = family_dir.name
# family CMake
family_mk = family_dir / "family.cmake"
# family.mk
if not family_mk.exists():
family_mk = family_dir / "family.mk"
mk_contents = family_mk.read_text()
# Find the mcu, first in family mk then board mk
@ -66,3 +75,48 @@ def skip_example(example, board):
"family:" + family in onlys)
return False
def build_example(example, board):
start_time = time.monotonic()
flash_size = "-"
sram_size = "-"
# succeeded, failed, skipped
ret = [0, 0, 0]
# Check if board is skipped
if skip_example(example, board):
status = SKIPPED
ret[2] = 1
print(build_format.format(example, board, status, '-', flash_size, sram_size))
else:
build_result = subprocess.run("make -j -C examples/{} BOARD={} all".format(example, board), shell=True,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
if build_result.returncode == 0:
status = SUCCEEDED
ret[0] = 1
(flash_size, sram_size) = build_size(example, board)
subprocess.run("make -j -C examples/{} BOARD={} copy-artifact".format(example, board), shell=True,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
else:
status = FAILED
ret[1] = 1
build_duration = time.monotonic() - start_time
print(build_format.format(example, board, status, "{:.2f}s".format(build_duration), flash_size, sram_size))
if build_result.returncode != 0:
print(build_result.stdout.decode("utf-8"))
return ret
def build_size(example, board):
elf_file = 'examples/{}/_build/{}/*.elf'.format(example, board)
size_output = subprocess.run('size {}'.format(elf_file), shell=True, stdout=subprocess.PIPE).stdout.decode("utf-8")
size_list = size_output.split('\n')[1].split('\t')
flash_size = int(size_list[0])
sram_size = int(size_list[1]) + int(size_list[2])
return (flash_size, sram_size)

25
tools/get_dependencies.py Normal file
View File

@ -0,0 +1,25 @@
import os
import sys
import subprocess
# dependency lookup (ABC sorted)
# deps = {
# 'LPC11UXX' : [ [] ]
# }
def get_family_dep(family):
for entry in os.scandir("hw/bsp/{}/boards".format(family)):
if entry.is_dir():
result = subprocess.run("make -C examples/device/board_test BOARD={} get-deps".format(entry.name),
shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
print(result.stdout.decode("utf-8"))
return result.returncode
status = 0
all_family = sys.argv[1:]
for f in all_family:
status += get_family_dep(f)
sys.exit(status)