From 1af5e97d8ded39947d5ccab21b9947fe362a105f Mon Sep 17 00:00:00 2001 From: Bin Meng Date: Sun, 27 Oct 2019 05:19:40 -0700 Subject: [PATCH 01/14] tools: image.h: Use portable uint32_t instead of linux-specific __be32 __be32 has Linux kernel specific __attribute__((bitwise)) which is not portable. Use uint32_t instead. Signed-off-by: Bin Meng --- include/image.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/include/image.h b/include/image.h index c1065c06f9b..f4d2aaf53e8 100644 --- a/include/image.h +++ b/include/image.h @@ -319,13 +319,13 @@ enum { * all data in network byte order (aka natural aka bigendian). */ typedef struct image_header { - __be32 ih_magic; /* Image Header Magic Number */ - __be32 ih_hcrc; /* Image Header CRC Checksum */ - __be32 ih_time; /* Image Creation Timestamp */ - __be32 ih_size; /* Image Data Size */ - __be32 ih_load; /* Data Load Address */ - __be32 ih_ep; /* Entry Point Address */ - __be32 ih_dcrc; /* Image Data CRC Checksum */ + uint32_t ih_magic; /* Image Header Magic Number */ + uint32_t ih_hcrc; /* Image Header CRC Checksum */ + uint32_t ih_time; /* Image Creation Timestamp */ + uint32_t ih_size; /* Image Data Size */ + uint32_t ih_load; /* Data Load Address */ + uint32_t ih_ep; /* Entry Point Address */ + uint32_t ih_dcrc; /* Image Data CRC Checksum */ uint8_t ih_os; /* Operating System */ uint8_t ih_arch; /* CPU architecture */ uint8_t ih_type; /* Image Type */ From a7c9a65ed9feee937c6b40a971d03e9c2f8447ca Mon Sep 17 00:00:00 2001 From: Bin Meng Date: Sun, 27 Oct 2019 05:19:41 -0700 Subject: [PATCH 02/14] tools: mtk_image.h: Use portable uintXX_t instead of linux-specific __leXX __leXX has Linux kernel specific __attribute__((bitwise)) which is not portable. Use corresponding uintXX_t instead. Signed-off-by: Bin Meng --- tools/mtk_image.h | 86 +++++++++++++++++++++++------------------------ 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/tools/mtk_image.h b/tools/mtk_image.h index 0a9eab372b9..4e78b3d0ff0 100644 --- a/tools/mtk_image.h +++ b/tools/mtk_image.h @@ -9,14 +9,14 @@ #ifndef _MTK_IMAGE_H #define _MTK_IMAGE_H -/* Device header definitions */ +/* Device header definitions, all fields are little-endian */ /* Header for NOR/SD/eMMC */ union gen_boot_header { struct { char name[12]; - __le32 version; - __le32 size; + uint32_t version; + uint32_t size; }; uint8_t pad[0x200]; @@ -32,14 +32,14 @@ union nand_boot_header { char name[12]; char version[4]; char id[8]; - __le16 ioif; - __le16 pagesize; - __le16 addrcycles; - __le16 oobsize; - __le16 pages_of_block; - __le16 numblocks; - __le16 writesize_shift; - __le16 erasesize_shift; + uint16_t ioif; + uint16_t pagesize; + uint16_t addrcycles; + uint16_t oobsize; + uint16_t pages_of_block; + uint16_t numblocks; + uint16_t writesize_shift; + uint16_t erasesize_shift; uint8_t dummy[60]; uint8_t ecc_parity[28]; }; @@ -54,14 +54,14 @@ union nand_boot_header { /* BootROM layout header */ struct brom_layout_header { char name[8]; - __le32 version; - __le32 header_size; - __le32 total_size; - __le32 magic; - __le32 type; - __le32 header_size_2; - __le32 total_size_2; - __le32 unused; + uint32_t version; + uint32_t header_size; + uint32_t total_size; + uint32_t magic; + uint32_t type; + uint32_t header_size_2; + uint32_t total_size_2; + uint32_t unused; }; #define BRLYT_NAME "BRLYT" @@ -90,8 +90,8 @@ struct gen_device_header { struct gfh_common_header { uint8_t magic[3]; uint8_t version; - __le16 size; - __le16 type; + uint16_t size; + uint16_t type; }; #define GFH_HEADER_MAGIC "MMM" @@ -106,17 +106,17 @@ struct gfh_common_header { struct gfh_file_info { struct gfh_common_header gfh; char name[12]; - __le32 unused; - __le16 file_type; + uint32_t unused; + uint16_t file_type; uint8_t flash_type; uint8_t sig_type; - __le32 load_addr; - __le32 total_size; - __le32 max_size; - __le32 hdr_size; - __le32 sig_size; - __le32 jump_offset; - __le32 processed; + uint32_t load_addr; + uint32_t total_size; + uint32_t max_size; + uint32_t hdr_size; + uint32_t sig_size; + uint32_t jump_offset; + uint32_t processed; }; #define GFH_FILE_INFO_NAME "FILE_INFO" @@ -129,16 +129,16 @@ struct gfh_file_info { struct gfh_bl_info { struct gfh_common_header gfh; - __le32 attr; + uint32_t attr; }; struct gfh_brom_cfg { struct gfh_common_header gfh; - __le32 cfg_bits; - __le32 usbdl_by_auto_detect_timeout_ms; + uint32_t cfg_bits; + uint32_t usbdl_by_auto_detect_timeout_ms; uint8_t unused[0x48]; - __le32 usbdl_by_kcol0_timeout_ms; - __le32 usbdl_by_flag_timeout_ms; + uint32_t usbdl_by_kcol0_timeout_ms; + uint32_t usbdl_by_flag_timeout_ms; uint32_t pad; }; @@ -157,15 +157,15 @@ struct gfh_anti_clone { uint8_t ac_b2k; uint8_t ac_b2c; uint16_t pad; - __le32 ac_offset; - __le32 ac_len; + uint32_t ac_offset; + uint32_t ac_len; }; struct gfh_brom_sec_cfg { struct gfh_common_header gfh; - __le32 cfg_bits; + uint32_t cfg_bits; char customer_name[0x20]; - __le32 pad; + uint32_t pad; }; #define BROM_SEC_CFG_JTAG_EN 1 @@ -184,11 +184,11 @@ struct gfh_header { union lk_hdr { struct { - __le32 magic; - __le32 size; + uint32_t magic; + uint32_t size; char name[32]; - __le32 loadaddr; - __le32 mode; + uint32_t loadaddr; + uint32_t mode; }; uint8_t data[512]; From afd3bf4318b0b3e845b5bc1f6763727229af9179 Mon Sep 17 00:00:00 2001 From: Bin Meng Date: Sun, 27 Oct 2019 05:19:42 -0700 Subject: [PATCH 03/14] tools: zynqmpbif: Use compiler builtin instead of linux-specific __swab32 __swab32() is a Linux specific macro defined in linux/swab.h. Let's use the compiler equivalent builtin function __builtin_bswap32() for better portability. Signed-off-by: Bin Meng --- tools/zynqmpbif.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/zynqmpbif.c b/tools/zynqmpbif.c index 8c47107c7b9..82ce0ac1a52 100644 --- a/tools/zynqmpbif.c +++ b/tools/zynqmpbif.c @@ -517,7 +517,7 @@ static int bif_add_bit(struct bif_entry *bf) debug("Bitstream Length: 0x%x\n", bitlen); for (i = 0; i < bitlen; i += sizeof(uint32_t)) { uint32_t *bitbin32 = (uint32_t *)&bitbin[i]; - *bitbin32 = __swab32(*bitbin32); + *bitbin32 = __builtin_bswap32(*bitbin32); } if (!bf->dest_dev) From 10289b4ed9f68bf3cd99d54acd4efdf7a96e281b Mon Sep 17 00:00:00 2001 From: Bin Meng Date: Sun, 27 Oct 2019 05:19:43 -0700 Subject: [PATCH 04/14] linux/types.h: Surround 'struct ustat' with __linux__ 'struct ustat' uses linux-specific typedefs to declare its memebers: __kernel_daddr_t and __kernel_ino_t. It is currently not used by any U-Boot codes, but when we build U-Boot tools for other platform like Windows, this becomes a problem. Let's surround it with __linux__. Signed-off-by: Bin Meng --- include/linux/types.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/linux/types.h b/include/linux/types.h index cc6f7cb39ee..51cb284bb88 100644 --- a/include/linux/types.h +++ b/include/linux/types.h @@ -151,12 +151,14 @@ typedef __u32 __bitwise __wsum; typedef unsigned __bitwise__ gfp_t; +#ifdef __linux__ struct ustat { __kernel_daddr_t f_tfree; __kernel_ino_t f_tinode; char f_fname[6]; char f_fpack[6]; }; +#endif #define DECLARE_BITMAP(name, bits) \ unsigned long name[BITS_TO_LONGS(bits)] From ccd4398fa71e305158ec53f4cd5b1f7fad27caaf Mon Sep 17 00:00:00 2001 From: Bin Meng Date: Sun, 27 Oct 2019 05:19:44 -0700 Subject: [PATCH 05/14] tools: ifwitool: Define __packed when it is not defined Some compilers may provide __packed define for us. Signed-off-by: Bin Meng --- tools/ifwitool.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/ifwitool.c b/tools/ifwitool.c index 2e020a82824..543e9d4e702 100644 --- a/tools/ifwitool.c +++ b/tools/ifwitool.c @@ -10,7 +10,9 @@ #include #include "os_support.h" +#ifndef __packed #define __packed __attribute__((packed)) +#endif #define KiB 1024 #define ALIGN(x, a) __ALIGN_MASK((x), (typeof(x))(a) - 1) #define __ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask)) From 7114d89e08a881285e09b8d36ce1f4dd8fe3d85b Mon Sep 17 00:00:00 2001 From: Bin Meng Date: Sun, 27 Oct 2019 05:19:45 -0700 Subject: [PATCH 06/14] doc: Add documentation for how to build U-Boot host tools This adds a reST document for how to build U-Boot host tools, including information for both Linux and Windows. Signed-off-by: Bin Meng --- doc/build/index.rst | 9 +++++++++ doc/build/tools.rst | 47 +++++++++++++++++++++++++++++++++++++++++++++ doc/index.rst | 11 +++++++++++ 3 files changed, 67 insertions(+) create mode 100644 doc/build/index.rst create mode 100644 doc/build/tools.rst diff --git a/doc/build/index.rst b/doc/build/index.rst new file mode 100644 index 00000000000..e4e34114af3 --- /dev/null +++ b/doc/build/index.rst @@ -0,0 +1,9 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +Build U-Boot +============ + +.. toctree:: + :maxdepth: 2 + + tools diff --git a/doc/build/tools.rst b/doc/build/tools.rst new file mode 100644 index 00000000000..c06f9152741 --- /dev/null +++ b/doc/build/tools.rst @@ -0,0 +1,47 @@ +.. SPDX-License-Identifier: GPL-2.0+ +.. sectionauthor:: Bin Meng + +Host tools +========== + +Building tools for Linux +------------------------ + +To allow distributions to distribute all possible tools in a generic way, +avoiding the need of specific tools building for each machine, a tools only +defconfig file is provided. + +Using this, we can build the tools by doing:: + + $ make tools-only_defconfig + $ make tools-only + +Building tools for Windows +-------------------------- +If you wish to generate Windows versions of the utilities in the tools directory +you can use MSYS2, a software distro and building platform for Windows. + +Download the MSYS2 installer from https://www.msys2.org. Make sure you have +installed all required packages below in order to build these host tools:: + + * gcc (9.1.0) + * make (4.2.1) + * bison (3.4.2) + * diffutils (3.7) + * openssl-devel (1.1.1.d) + +Note the version numbers in these parentheses above are the package versions +at the time being when writing this document. The MSYS2 installer tested is +http://repo.msys2.org/distrib/x86_64/msys2-x86_64-20190524.exe. + +There are 3 MSYS subsystems installed: MSYS2, MinGW32 and MinGW64. Each +subsystem provides an environment to build Windows applications. The MSYS2 +environment is for building POSIX compliant software on Windows using an +emulation layer. The MinGW32/64 subsystems are for building native Windows +applications using a linux toolchain (gcc, bash, etc), targeting respectively +32 and 64 bit Windows. + +Launch the MSYS2 shell of the MSYS2 environment, and do the following:: + + $ make tools-only_defconfig + $ make tools-only NO_SDL=1 diff --git a/doc/index.rst b/doc/index.rst index 458f0d2d0e0..206a04590fa 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -15,6 +15,17 @@ if you want to help out. .. toctree:: :maxdepth: 2 +User-oriented documentation +--------------------------- + +The following manuals are written for *users* of the U-Boot - those who are +trying to get it to work optimally on a given system. + +.. toctree:: + :maxdepth: 2 + + build/index + Unified Extensible Firmware (UEFI) ---------------------------------- From 1f35137997a25a14c734abd2c7a2fe9406944307 Mon Sep 17 00:00:00 2001 From: Bin Meng Date: Sun, 27 Oct 2019 05:19:46 -0700 Subject: [PATCH 07/14] Add .gitattributes for line endings When building U-Boot host tools for Windows from Microsoft Azure Pipelines, we see tons of weird warnings and errors emitted from every Kconfig files: Kconfig:6:warning: ignoring unsupported character '' Kconfig:6:warning: ignoring unsupported character '' Kconfig:8:warning: ignoring unsupported character '' Kconfig:9:warning: ignoring unsupported character '' Kconfig:10:warning: ignoring unsupported character '' Kconfig:10:warning: ignoring unsupported character '' Kconfig:13:warning: ignoring unsupported character '' arch/Kconfig:1:warning: ignoring unsupported character '' arch/Kconfig:2:warning: ignoring unsupported character '' arch/Kconfig:2:warning: ignoring unsupported character '' arch/Kconfig:4:warning: ignoring unsupported character '' ... After several rounds of experiments, it turns out this is caused by line endings. Historically, Linux and macOS used linefeed (LF) characters while Windows used a carriage return plus a linefeed (CRLF). When Azure Pipelines checks out the U-Boot repo, Git tries to compensate for the difference by automatically making lines end in CRLF in the working directory on Windows, which confuses the Kconfig file parsing logic. Fortunately Git provides a way for repos to tell Git not to do such automatical line endings conversion via .gitattributes file below: * text eol=lf Signed-off-by: Bin Meng --- .gitattributes | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000000..8560b794c0a --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Declare files that always have LF line endings on checkout +* text eol=lf From 4d90f6cd9813cb513303a91d8b53e1ee027324bd Mon Sep 17 00:00:00 2001 From: Bin Meng Date: Sun, 27 Oct 2019 05:19:47 -0700 Subject: [PATCH 08/14] tools: Avoid creating symbolic links for tools/version.h MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When building U-Boot host tools for Windows from Microsoft Azure Pipelines, the following errors were seen: HOSTCC tools/mkenvimage.o In file included from tools/mkenvimage.c:25: ./tools/version.h:1:1: error: expected identifier or ‘(’ before ‘.’ token 1 | ../include/version.h | ^ tools/mkenvimage.c: In function ‘main’: tools/mkenvimage.c:117:4: warning: implicit declaration of function ‘usage’ [-Wimplicit-function-declaration] 117 | usage(prg); | ^~~~~ tools/mkenvimage.c:120:35: error: ‘PLAIN_VERSION’ undeclared (first use in this function) 120 | printf("%s version %s\n", prg, PLAIN_VERSION); | ^~~~~~~~~~~~~ tools/mkenvimage.c:120:35: note: each undeclared identifier is reported only once for each function it appears in make[2]: *** [scripts/Makefile.host:114: tools/mkenvimage.o] Error 1 It turns out tools/version.h is a symbolic link and with Windows default settings it is unsupported hence the actual content of tools/version.h is not what file include/version.h has, but the the linked file path, which breaks the build. To fix this, remove the symbolic links for tools/version.h. Instead we perform a copy from include/version.h during the build. Signed-off-by: Bin Meng --- Makefile | 9 ++++++--- tools/.gitignore | 1 + tools/version.h | 1 - 3 files changed, 7 insertions(+), 4 deletions(-) delete mode 120000 tools/version.h diff --git a/Makefile b/Makefile index cbacf1cfe2c..e78b3179b13 100644 --- a/Makefile +++ b/Makefile @@ -1837,11 +1837,14 @@ checkarmreloc: u-boot false; \ fi -envtools: scripts_basic $(version_h) $(timestamp_h) +tools/version.h: include/version.h + $(call if_changed,copy) + +envtools: scripts_basic $(version_h) $(timestamp_h) tools/version.h $(Q)$(MAKE) $(build)=tools/env tools-only: export TOOLS_ONLY=y -tools-only: scripts_basic $(version_h) $(timestamp_h) +tools-only: scripts_basic $(version_h) $(timestamp_h) tools/version.h $(Q)$(MAKE) $(build)=tools tools-all: export HOST_TOOLS_ALL=y @@ -1869,7 +1872,7 @@ CLEAN_DIRS += $(MODVERDIR) \ $(foreach d, spl tpl, $(patsubst %,$d/%, \ $(filter-out include, $(shell ls -1 $d 2>/dev/null)))) -CLEAN_FILES += include/bmp_logo.h include/bmp_logo_data.h \ +CLEAN_FILES += include/bmp_logo.h include/bmp_logo_data.h tools/version.h \ boot* u-boot* MLO* SPL System.map fit-dtb.blob* # Directories & files removed with 'make mrproper' diff --git a/tools/.gitignore b/tools/.gitignore index bd03d32f684..d0176a7283f 100644 --- a/tools/.gitignore +++ b/tools/.gitignore @@ -31,4 +31,5 @@ /spl_size_limit /sunxi-spl-image-builder /ubsha1 +/version.h /xway-swap-bytes diff --git a/tools/version.h b/tools/version.h deleted file mode 120000 index bb576071e84..00000000000 --- a/tools/version.h +++ /dev/null @@ -1 +0,0 @@ -../include/version.h \ No newline at end of file From d2e680fa557539e324928838781bd38245caeb98 Mon Sep 17 00:00:00 2001 From: Bin Meng Date: Sun, 27 Oct 2019 05:19:48 -0700 Subject: [PATCH 09/14] Add Microsoft Azure Pipelines configuration Microsoft Azure Pipelines [1] provides unlimited CI/CD minutes and 10 parallel jobs to every open source project for free. This adds a configuration file for Azure Pipelines to utilize the free Windows VM hosted by Microsoft to ensure no build broken in building U-Boot host tools for Windows. [1] https://azure.microsoft.com/en-us/blog/announcing-azure-pipelines-with-unlimited-ci-cd-minutes-for-open-source/ Signed-off-by: Bin Meng --- .azure-pipelines.yml | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 .azure-pipelines.yml diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml new file mode 100644 index 00000000000..6c6b24e4984 --- /dev/null +++ b/.azure-pipelines.yml @@ -0,0 +1,38 @@ +variables: + windows_vm: vs2015-win2012r2 + +jobs: + - job: tools_only_windows + displayName: 'Ensure host tools build for Windows' + pool: + vmImage: $(windows_vm) + strategy: + matrix: + i686: + MSYS_DIR: msys32 + BASE_REPO: msys2-ci-base-i686 + x86_64: + MSYS_DIR: msys64 + BASE_REPO: msys2-ci-base + steps: + - script: | + git clone https://github.com/msys2/$(BASE_REPO).git %CD:~0,2%\$(MSYS_DIR) + displayName: 'Install MSYS2' + - script: | + set PATH=%CD:~0,2%\$(MSYS_DIR)\usr\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem + %CD:~0,2%\$(MSYS_DIR)\usr\bin\pacman --noconfirm -Syyuu + displayName: 'Update MSYS2' + - script: | + set PATH=%CD:~0,2%\$(MSYS_DIR)\usr\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem + %CD:~0,2%\$(MSYS_DIR)\usr\bin\pacman --noconfirm --needed -S make gcc bison diffutils openssl-devel + displayName: 'Install Toolchain' + - script: | + set PATH=C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem + echo make tools-only_defconfig tools-only NO_SDL=1 > build-tools.sh + %CD:~0,2%\$(MSYS_DIR)\usr\bin\bash -lc "bash build-tools.sh" + displayName: 'Build Host Tools' + env: + # Tell MSYS2 we need a POSIX emulation layer + MSYSTEM: MSYS + # Tell MSYS2 not to ‘cd’ our startup directory to HOME + CHERE_INVOKING: yes From bc750bca1246de09ca653141c1add7e10c04a37c Mon Sep 17 00:00:00 2001 From: Bin Meng Date: Mon, 28 Oct 2019 07:24:59 -0700 Subject: [PATCH 10/14] tools: buildman: Honor output directory when generating boards.cfg buildman always generates boards.cfg in the U-Boot source tree. When '-o' is given, we should generate boards.cfg to the given output directory. Signed-off-by: Bin Meng Reviewed-by: Simon Glass --- tools/buildman/control.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/buildman/control.py b/tools/buildman/control.py index fcf531c5f14..9787b867476 100644 --- a/tools/buildman/control.py +++ b/tools/buildman/control.py @@ -201,14 +201,14 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, # Work out what subset of the boards we are building if not boards: - board_file = os.path.join(options.git, 'boards.cfg') - status = subprocess.call([os.path.join(options.git, - 'tools/genboardscfg.py')]) + board_file = os.path.join(options.output_dir, 'boards.cfg') + genboardscfg = os.path.join(options.git, 'tools/genboardscfg.py') + status = subprocess.call([genboardscfg, '-o', board_file]) if status != 0: - sys.exit("Failed to generate boards.cfg") + sys.exit("Failed to generate boards.cfg") boards = board.Boards() - boards.ReadBoards(os.path.join(options.git, 'boards.cfg')) + boards.ReadBoards(board_file) exclude = [] if options.exclude: From 578b6372030f670151f109f00a87c0b9b281c998 Mon Sep 17 00:00:00 2001 From: Bin Meng Date: Mon, 28 Oct 2019 07:25:00 -0700 Subject: [PATCH 11/14] tools: buildman: Remove useless mkdir() in Make() in test.py In the 'Make' function, the codes tries to create a directory if current stage is 'build'. But the directory isn't used at all anywhere. Signed-off-by: Bin Meng Reviewed-by: Simon Glass --- tools/buildman/test.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tools/buildman/test.py b/tools/buildman/test.py index de02f61be65..ed99b9375c4 100644 --- a/tools/buildman/test.py +++ b/tools/buildman/test.py @@ -156,14 +156,6 @@ class TestBuild(unittest.TestCase): result.return_code = commit.return_code result.stderr = (''.join(commit.error_list) % {'basedir' : base_dir + '/.bm-work/00/'}) - if stage == 'build': - target_dir = None - for arg in args: - if arg.startswith('O='): - target_dir = arg[2:] - - if not os.path.isdir(target_dir): - os.mkdir(target_dir) result.combined = result.stdout + result.stderr return result From 6cdd177ca9092d1ee094dd3709e25d116e329ca0 Mon Sep 17 00:00:00 2001 From: Bin Meng Date: Mon, 28 Oct 2019 07:25:01 -0700 Subject: [PATCH 12/14] arm: mvebu: Avoid generating kwbimage.cfg in the source tree At present some boards generate kwbimage.cfg in the source tree during the build. This breaks buildman testing on some systems where the source tree is read-only. Update makefile rules to generate it in the build tree instead. Note some other boards have the kwbimage.cfg file written in advance, hence we need check if the file exists in the build tree first, otherwise we fall back to one in the source tree. Signed-off-by: Bin Meng Reviewed-by: Chris Packham Reviewed-by: Stefan Roese Reviewed-by: Simon Glass --- Makefile | 15 +++++++++++++-- arch/arm/mach-mvebu/Makefile | 4 ++-- board/Marvell/db-88f6281-bp/Makefile | 4 ++-- board/Marvell/db-xc3-24g4xg/Makefile | 4 ++-- board/mikrotik/crs305-1g-4s/Makefile | 4 ++-- 5 files changed, 21 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index e78b3179b13..61b94c5a6e2 100644 --- a/Makefile +++ b/Makefile @@ -1276,10 +1276,21 @@ endif MKIMAGEFLAGS_u-boot-dtb.img = $(MKIMAGEFLAGS_u-boot.img) -MKIMAGEFLAGS_u-boot.kwb = -n $(srctree)/$(CONFIG_SYS_KWD_CONFIG:"%"=%) \ +# Some boards have the kwbimage.cfg file written in advance, while some +# other boards generate it on the fly during the build in the build tree. +# Let's check if the file exists in the build tree first, otherwise we +# fall back to use the one in the source tree. +KWD_CONFIG_FILE = $(shell \ + if [ -f $(objtree)/$(CONFIG_SYS_KWD_CONFIG:"%"=%) ]; then \ + echo -n $(objtree)/$(CONFIG_SYS_KWD_CONFIG:"%"=%); \ + else \ + echo -n $(srctree)/$(CONFIG_SYS_KWD_CONFIG:"%"=%); \ + fi) + +MKIMAGEFLAGS_u-boot.kwb = -n $(KWD_CONFIG_FILE) \ -T kwbimage -a $(CONFIG_SYS_TEXT_BASE) -e $(CONFIG_SYS_TEXT_BASE) -MKIMAGEFLAGS_u-boot-spl.kwb = -n $(srctree)/$(CONFIG_SYS_KWD_CONFIG:"%"=%) \ +MKIMAGEFLAGS_u-boot-spl.kwb = -n $(KWD_CONFIG_FILE) \ -T kwbimage -a $(CONFIG_SYS_TEXT_BASE) -e $(CONFIG_SYS_TEXT_BASE) \ $(if $(KEYDIR),-k $(KEYDIR)) diff --git a/arch/arm/mach-mvebu/Makefile b/arch/arm/mach-mvebu/Makefile index 8228a17972f..b73952044de 100644 --- a/arch/arm/mach-mvebu/Makefile +++ b/arch/arm/mach-mvebu/Makefile @@ -58,10 +58,10 @@ KWB_REPLACE += SEC_FUSE_DUMP KWB_CFG_SEC_FUSE_DUMP = a38x endif -$(src)/kwbimage.cfg: $(src)/kwbimage.cfg.in include/autoconf.mk \ +$(obj)/kwbimage.cfg: $(src)/kwbimage.cfg.in include/autoconf.mk \ include/config/auto.conf $(Q)sed -ne '$(foreach V,$(KWB_REPLACE),s/^#@$(V)/$(V) $(KWB_CFG_$(V))/;)p' \ - <$< >$(dir $<)$(@F) + <$< >$(dir $@)$(@F) endif # CONFIG_SPL_BUILD obj-y += gpio.o diff --git a/board/Marvell/db-88f6281-bp/Makefile b/board/Marvell/db-88f6281-bp/Makefile index e6aa7e39a07..003e9f66d18 100644 --- a/board/Marvell/db-88f6281-bp/Makefile +++ b/board/Marvell/db-88f6281-bp/Makefile @@ -4,9 +4,9 @@ obj-y := db-88f6281-bp.o extra-y := kwbimage.cfg quiet_cmd_sed = SED $@ - cmd_sed = sed $(SEDFLAGS_$(@F)) $< >$(dir $<)$(@F) + cmd_sed = sed $(SEDFLAGS_$(@F)) $< >$(dir $@)$(@F) SEDFLAGS_kwbimage.cfg = -e "s/^\#@BOOT_FROM.*/BOOT_FROM $(if $(CONFIG_CMD_NAND),nand,spi)/" -$(src)/kwbimage.cfg: $(src)/kwbimage.cfg.in include/autoconf.mk \ +$(obj)/kwbimage.cfg: $(src)/kwbimage.cfg.in include/autoconf.mk \ include/config/auto.conf $(call if_changed,sed) diff --git a/board/Marvell/db-xc3-24g4xg/Makefile b/board/Marvell/db-xc3-24g4xg/Makefile index 4dd57902d47..24e82000075 100644 --- a/board/Marvell/db-xc3-24g4xg/Makefile +++ b/board/Marvell/db-xc3-24g4xg/Makefile @@ -6,9 +6,9 @@ obj-y := db-xc3-24g4xg.o extra-y := kwbimage.cfg quiet_cmd_sed = SED $@ - cmd_sed = sed $(SEDFLAGS_$(@F)) $< >$(dir $<)$(@F) + cmd_sed = sed $(SEDFLAGS_$(@F)) $< >$(dir $@)$(@F) SEDFLAGS_kwbimage.cfg =-e "s|^BINARY.*|BINARY $(srctree)/$(@D)/binary.0 0000005b 00000068|" -$(src)/kwbimage.cfg: $(src)/kwbimage.cfg.in include/autoconf.mk \ +$(obj)/kwbimage.cfg: $(src)/kwbimage.cfg.in include/autoconf.mk \ include/config/auto.conf $(call if_changed,sed) diff --git a/board/mikrotik/crs305-1g-4s/Makefile b/board/mikrotik/crs305-1g-4s/Makefile index 895331beb86..c03f534e485 100644 --- a/board/mikrotik/crs305-1g-4s/Makefile +++ b/board/mikrotik/crs305-1g-4s/Makefile @@ -6,9 +6,9 @@ obj-y := crs305-1g-4s.o extra-y := kwbimage.cfg quiet_cmd_sed = SED $@ - cmd_sed = sed $(SEDFLAGS_$(@F)) $< >$(dir $<)$(@F) + cmd_sed = sed $(SEDFLAGS_$(@F)) $< >$(dir $@)$(@F) SEDFLAGS_kwbimage.cfg =-e "s|^BINARY.*|BINARY $(srctree)/$(@D)/binary.0 0000005b 00000068|" -$(src)/kwbimage.cfg: $(src)/kwbimage.cfg.in include/autoconf.mk \ +$(obj)/kwbimage.cfg: $(src)/kwbimage.cfg.in include/autoconf.mk \ include/config/auto.conf $(call if_changed,sed) From d3c369d1967bec9f4efc7e02ceb891da71d1f500 Mon Sep 17 00:00:00 2001 From: Bin Meng Date: Mon, 28 Oct 2019 07:25:02 -0700 Subject: [PATCH 13/14] .travis.yml: Remove the unneeded '&' for ls20xx buildman Signed-off-by: Bin Meng Reviewed-by: Tom Rini --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a3e7451bcb1..3aaed933465 100644 --- a/.travis.yml +++ b/.travis.yml @@ -200,7 +200,7 @@ matrix: - BUILDMAN="freescale&aarch64&ls108" - name: "buildman NXP AArch64 LS20xx" env: - - BUILDMAN="freescale&aarch64&&ls20" + - BUILDMAN="freescale&aarch64&ls20" - name: "buildman NXP AArch64 LX216x" env: - BUILDMAN="freescale&aarch64&lx216" From bf275222ab80650a9dbc559ba42e52d93ff86af4 Mon Sep 17 00:00:00 2001 From: Bin Meng Date: Mon, 28 Oct 2019 07:25:03 -0700 Subject: [PATCH 14/14] Bring all testings in gitlab and travis CI to Azure Pipelines This expands current Azure Pipelines Windows host tools build testing to cover all the CI testing in gitlab and travis CI. Note for some unknown reason, the 'container' cannot be used for any jobs that have buildman, for buildman does not exit properly and hangs the job forever. As a workaround, we manually call docker to run the image to perform the CI tasks. A complete run on Azure Pipelines takes about 2 hours and 10 minutes. Signed-off-by: Bin Meng --- .azure-pipelines.yml | 382 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 382 insertions(+) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 6c6b24e4984..d476d8d0e94 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -1,5 +1,12 @@ variables: windows_vm: vs2015-win2012r2 + ubuntu_vm: ubuntu-18.04 + ci_runner_image: trini/u-boot-gitlab-ci-runner:bionic-20190912.1-03Oct2019 + # Add '-u 0' options for Azure pipelines, otherwise we get "permission + # denied" error when it tries to "useradd -m -u 1001 vsts_azpcontainer", + # since our $(ci_runner_image) user is not root. + container_option: -u 0 + work_dir: /u jobs: - job: tools_only_windows @@ -36,3 +43,378 @@ jobs: MSYSTEM: MSYS # Tell MSYS2 not to ‘cd’ our startup directory to HOME CHERE_INVOKING: yes + + - job: cppcheck + displayName: 'Static code analysis with cppcheck' + pool: + vmImage: $(ubuntu_vm) + container: + image: $(ci_runner_image) + options: $(container_option) + steps: + - script: cppcheck --force --quiet --inline-suppr . + + - job: todo + displayName: 'Search for TODO within source tree' + pool: + vmImage: $(ubuntu_vm) + container: + image: $(ci_runner_image) + options: $(container_option) + steps: + - script: grep -r TODO . + - script: grep -r FIXME . + - script: grep -r HACK . | grep -v HACKKIT + + - job: sloccount + displayName: 'Some statistics about the code base' + pool: + vmImage: $(ubuntu_vm) + container: + image: $(ci_runner_image) + options: $(container_option) + steps: + - script: sloccount . + + - job: maintainers + displayName: 'Ensure all configs have MAINTAINERS entries' + pool: + vmImage: $(ubuntu_vm) + container: + image: $(ci_runner_image) + options: $(container_option) + steps: + - script: | + if [ `./tools/genboardscfg.py -f 2>&1 | wc -l` -ne 0 ]; then exit 1; fi + + - job: tools_only + displayName: 'Ensure host tools build' + pool: + vmImage: $(ubuntu_vm) + container: + image: $(ci_runner_image) + options: $(container_option) + steps: + - script: | + make tools-only_config tools-only -j$(nproc) + + - job: envtools + displayName: 'Ensure env tools build' + pool: + vmImage: $(ubuntu_vm) + container: + image: $(ci_runner_image) + options: $(container_option) + steps: + - script: | + make tools-only_config envtools -j$(nproc) + + - job: utils + displayName: 'Run binman, buildman, dtoc and patman testsuites' + pool: + vmImage: $(ubuntu_vm) + steps: + - script: | + cat << EOF > build.sh + set -ex + cd ${WORK_DIR} + EOF + cat << "EOF" >> build.sh + git config --global user.name "Azure Pipelines" + git config --global user.email bmeng.cn@gmail.com + export USER=azure + virtualenv /tmp/venv + . /tmp/venv/bin/activate + pip install pyelftools + export UBOOT_TRAVIS_BUILD_DIR=/tmp/.bm-work/sandbox_spl + export PYTHONPATH=${UBOOT_TRAVIS_BUILD_DIR}/scripts/dtc/pylibfdt + export PATH=${UBOOT_TRAVIS_BUILD_DIR}/scripts/dtc:${PATH} + ./tools/buildman/buildman -o /tmp -P sandbox_spl + ./tools/binman/binman --toolpath ${UBOOT_TRAVIS_BUILD_DIR}/tools test + ./tools/buildman/buildman -t + ./tools/dtoc/dtoc -t + ./tools/patman/patman --test + EOF + cat build.sh + # We cannot use "container" like other jobs above, as buildman + # seems to hang forever with pre-configured "container" environment + docker run -v $PWD:$(work_dir) $(ci_runner_image) /bin/bash $(work_dir)/build.sh + + - job: test_py + displayName: 'test.py' + pool: + vmImage: $(ubuntu_vm) + strategy: + matrix: + sandbox: + TEST_PY_BD: "sandbox" + BUILDMAN: "^sandbox$" + sandbox_spl: + TEST_PY_BD: "sandbox_spl" + TEST_PY_TEST_SPEC: "test_ofplatdata" + BUILDMAN: "^sandbox_spl$" + sandbox_flattree: + TEST_PY_BD: "sandbox_flattree" + BUILDMAN: "^sandbox_flattree$" + evb_ast2500: + TEST_PY_BD: "evb-ast2500" + TEST_PY_ID: "--id qemu" + BUILDMAN: "^evb-ast2500$" + vexpress_ca15_tc2: + TEST_PY_BD: "vexpress_ca15_tc2" + TEST_PY_ID: "--id qemu" + BUILDMAN: "^vexpress_ca15_tc2$" + vexpress_ca9x4: + TEST_PY_BD: "vexpress_ca9x4" + TEST_PY_ID: "--id qemu" + BUILDMAN: "^vexpress_ca9x4$" + integratorcp_cm926ejs: + TEST_PY_BD: "integratorcp_cm926ejs" + TEST_PY_ID: "--id qemu" + TEST_PY_TEST_SPEC: "not sleep" + BUILDMAN: "^integratorcp_cm926ejs$" + qemu_arm: + TEST_PY_BD: "qemu_arm" + TEST_PY_TEST_SPEC: "not sleep" + BUILDMAN: "^qemu_arm$" + qemu_arm64: + TEST_PY_BD: "qemu_arm64" + TEST_PY_TEST_SPEC: "not sleep" + BUILDMAN: "^qemu_arm64$" + qemu_mips: + TEST_PY_BD: "qemu_mips" + TEST_PY_TEST_SPEC: "not sleep" + BUILDMAN: "^qemu_mips$" + qemu_mipsel: + TEST_PY_BD: "qemu_mipsel" + TEST_PY_TEST_SPEC: "not sleep" + BUILDMAN: "^qemu_mipsel$" + qemu_mips64: + TEST_PY_BD: "qemu_mips64" + TEST_PY_TEST_SPEC: "not sleep" + BUILDMAN: "^qemu_mips64$" + qemu_mips64el: + TEST_PY_BD: "qemu_mips64el" + TEST_PY_TEST_SPEC: "not sleep" + BUILDMAN: "^qemu_mips64el$" + qemu_ppce500: + TEST_PY_BD: "qemu-ppce500" + TEST_PY_TEST_SPEC: "not sleep" + BUILDMAN: "^qemu-ppce500$" + qemu_riscv64: + TEST_PY_BD: "qemu-riscv64" + TEST_PY_TEST_SPEC: "not sleep" + BUILDMAN: "^qemu-riscv64$" + qemu_x86: + TEST_PY_BD: "qemu-x86" + TEST_PY_TEST_SPEC: "not sleep" + BUILDMAN: "^qemu-x86$" + qemu_x86_64: + TEST_PY_BD: "qemu-x86_64" + TEST_PY_TEST_SPEC: "not sleep" + BUILDMAN: "^qemu-x86_64$" + zynq_zc702: + TEST_PY_BD: "zynq_zc702" + TEST_PY_ID: "--id qemu" + TEST_PY_TEST_SPEC: "not sleep" + BUILDMAN: "^zynq_zc702$" + xilinx_versal_virt: + TEST_PY_BD: "xilinx_versal_virt" + TEST_PY_ID: "--id qemu" + TEST_PY_TEST_SPEC: "not sleep" + BUILDMAN: "^xilinx_versal_virt$" + xtfpga: + TEST_PY_BD: "xtfpga" + TEST_PY_ID: "--id qemu" + TEST_PY_TEST_SPEC: "not sleep" + BUILDMAN: "^xtfpga$" + steps: + - script: | + cat << EOF > test.sh + set -ex + # make environment variables available as tests are running inside a container + export WORK_DIR="${WORK_DIR}" + export TEST_PY_BD="${TEST_PY_BD}" + export TEST_PY_ID="${TEST_PY_ID}" + export TEST_PY_TEST_SPEC="${TEST_PY_TEST_SPEC}" + export BUILDMAN="${BUILDMAN}" + EOF + cat << "EOF" >> test.sh + # the below corresponds to .gitlab-ci.yml "before_script" + cd ${WORK_DIR} + git clone --depth=1 git://github.com/swarren/uboot-test-hooks.git /tmp/uboot-test-hooks + ln -s travis-ci /tmp/uboot-test-hooks/bin/`hostname` + ln -s travis-ci /tmp/uboot-test-hooks/py/`hostname` + virtualenv /tmp/venv + . /tmp/venv/bin/activate + pip install pytest==2.8.7 + pip install python-subunit + pip install coverage + grub-mkimage --prefix=\"\" -o ~/grub_x86.efi -O i386-efi normal echo lsefimmap lsefi lsefisystab efinet tftp minicmd + grub-mkimage --prefix=\"\" -o ~/grub_x64.efi -O x86_64-efi normal echo lsefimmap lsefi lsefisystab efinet tftp minicmd + mkdir ~/grub2-arm + cd ~/grub2-arm; wget -O - http://download.opensuse.org/ports/armv7hl/distribution/leap/42.2/repo/oss/suse/armv7hl/grub2-arm-efi-2.02~beta2-87.1.armv7hl.rpm | rpm2cpio | cpio -di + mkdir ~/grub2-arm64 + cd ~/grub2-arm64; wget -O - http://download.opensuse.org/ports/aarch64/distribution/leap/42.2/repo/oss/suse/aarch64/grub2-arm64-efi-2.02~beta2-87.1.aarch64.rpm | rpm2cpio | cpio -di + # the below corresponds to .gitlab-ci.yml "script" + cd ${WORK_DIR} + if [[ "${BUILDMAN}" != "" ]]; then + ret=0; + tools/buildman/buildman -o /tmp -P -E ${BUILDMAN} ${OVERRIDE} || ret=$?; + if [[ $ret -ne 0 && $ret -ne 129 ]]; then + tools/buildman/buildman -o /tmp -sdeP ${BUILDMAN}; + exit $ret; + fi; + fi + export UBOOT_TRAVIS_BUILD_DIR=/tmp/.bm-work/${TEST_PY_BD}; + export PATH=/opt/qemu/bin:/tmp/uboot-test-hooks/bin:/usr/bin:/bin; + export PYTHONPATH=/tmp/uboot-test-hooks/py/travis-ci; + if [[ "${TEST_PY_BD}" != "" ]]; then + ./test/py/test.py --bd ${TEST_PY_BD} ${TEST_PY_ID} -k "${TEST_PY_TEST_SPEC:-not a_test_which_does_not_exist}" --build-dir "$UBOOT_TRAVIS_BUILD_DIR"; + ret=$?; + if [[ $ret -ne 0 ]]; then + exit $ret; + fi; + fi + # the below corresponds to .gitlab-ci.yml "after_script" + rm -rf ~/grub2* /tmp/uboot-test-hooks /tmp/venv + EOF + cat test.sh + # make current directory writeable to uboot user inside the container + # as sandbox testing need create files like spi flash images, etc. + # (TODO: clean up this in the future) + chmod 777 . + docker run -v $PWD:$(work_dir) $(ci_runner_image) /bin/bash $(work_dir)/test.sh + + - job: build_the_world + displayName: 'Build the World' + pool: + vmImage: $(ubuntu_vm) + strategy: + # Use almost the same target division in .travis.yml, only merged + # 4 small build jobs (arc/microblaze/nds32/xtensa) into one. + matrix: + arc_microblaze_nds32_xtensa: + BUILDMAN: "arc microblaze nds32 xtensa" + arm11_arm7_arm920t_arm946es: + BUILDMAN: "arm11 arm7 arm920t arm946es" + arm926ejs: + BUILDMAN: "arm926ejs -x freescale,siemens,at91,kirkwood,spear,omap" + at91_non_armv7: + BUILDMAN: "at91 -x armv7" + at91_non_arm926ejs: + BUILDMAN: "at91 -x arm926ejs" + boundary_engicam_toradex: + BUILDMAN: "boundary engicam toradex" + arm_bcm: + BUILDMAN: "bcm -x mips" + nxp_arm32: + BUILDMAN: "freescale -x powerpc,m68k,aarch64" + nxp_aarch64_ls101x: + BUILDMAN: "freescale&aarch64&ls101" + nxp_aarch64_ls102x: + BUILDMAN: "freescale&aarch64&ls102" + nxp_aarch64_ls104x: + BUILDMAN: "freescale&aarch64&ls104" + nxp_aarch64_ls108x: + BUILDMAN: "freescale&aarch64&ls108" + nxp_aarch64_ls20xx: + BUILDMAN: "freescale&aarch64&ls20" + nxp_aarch64_lx216x: + BUILDMAN: "freescale&aarch64&lx216" + imx6: + BUILDMAN: "mx6 -x boundary,engicam,freescale,technexion,toradex" + imx: + BUILDMAN: "mx -x mx6,freescale,technexion,toradex" + keystone2_keystone3: + BUILDMAN: "k2 k3" + samsung_socfpga: + BUILDMAN: "samsung socfpga" + spear: + BUILDMAN: "spear" + sun4i: + BUILDMAN: "sun4i" + sun5i: + BUILDMAN: "sun5i" + sun6i: + BUILDMAN: "sun6i" + sun7i: + BUILDMAN: "sun7i" + sun8i_32bit: + BUILDMAN: "sun8i&armv7" + sun8i_64bit: + BUILDMAN: "sun8i&aarch64" + sun9i: + BUILDMAN: "sun9i" + sun50i: + BUILDMAN: "sun50i" + arm_catch_all: + BUILDMAN: "arm -x arm11,arm7,arm9,aarch64,at91,bcm,freescale,kirkwood,mvebu,siemens,tegra,uniphier,mx,samsung,sunxi,am33xx,omap,rockchip,toradex,socfpga,k2,k3,zynq" + sandbox_x86: + BUILDMAN: "sandbox x86" + technexion: + BUILDMAN: "technexion" + kirkwood: + BUILDMAN: "kirkwood" + mvebu: + BUILDMAN: "mvebu" + m68k: + BUILDMAN: "m68k" + mips: + BUILDMAN: "mips" + non_fsl_ppc: + BUILDMAN: "powerpc -x freescale" + mpc85xx_freescale: + BUILDMAN: "mpc85xx&freescale -x t208xrdb -x t4qds -x t102* -x p1_p2_rdb_pc -x p1010rdb -x corenet_ds -x b4860qds -x bsc91*" + t208xrdb_corenet_ds: + BUILDMAN: "t208xrdb corenet_ds" + fsl_ppc: + BUILDMAN: "t4qds b4860qds mpc83xx&freescale mpc86xx&freescale" + t102x: + BUILDMAN: "t102*" + p1_p2_rdb_pc: + BUILDMAN: "p1_p2_rdb_pc" + p1010rdb_bsc91: + BUILDMAN: "p1010rdb bsc91" + siemens: + BUILDMAN: "siemens" + tegra: + BUILDMAN: "tegra -x toradex" + am33xx_no_siemens: + BUILDMAN: "am33xx -x siemens" + omap: + BUILDMAN: "omap" + uniphier: + BUILDMAN: "uniphier" + aarch64_catch_all: + BUILDMAN: "aarch64 -x bcm,k3,tegra,ls1,ls2,mvebu,uniphier,sunxi,samsung,rockchip,versal,zynq" + rockchip: + BUILDMAN: "rockchip" + sh: + BUILDMAN: "sh -x arm" + zynq: + BUILDMAN: "zynq&armv7" + zynqmp_versal: + BUILDMAN: "versal|zynqmp&aarch64" + riscv: + BUILDMAN: "riscv" + steps: + - script: | + cat << EOF > build.sh + set -ex + cd ${WORK_DIR} + # make environment variables available as tests are running inside a container + export BUILDMAN="${BUILDMAN}" + EOF + cat << "EOF" >> build.sh + if [[ "${BUILDMAN}" != "" ]]; then + ret=0; + tools/buildman/buildman -o /tmp -P -E ${BUILDMAN} ${OVERRIDE} || ret=$?; + if [[ $ret -ne 0 && $ret -ne 129 ]]; then + tools/buildman/buildman -o /tmp -sdeP ${BUILDMAN}; + exit $ret; + fi; + fi + EOF + cat build.sh + docker run -v $PWD:$(work_dir) $(ci_runner_image) /bin/bash $(work_dir)/build.sh