commit c03c962f6fc27ff6667671ec6dcee91904528082 Author: José Moreira Date: Sat Sep 8 17:22:54 2018 +0100 Initial commit diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..43c4a8c --- /dev/null +++ b/.editorconfig @@ -0,0 +1,20 @@ +# http://editorconfig.org +root = true + +[*] +charset = utf-8 +end_of_line = lf +indent_size = 2 +indent_style = space +max_line_length = 80 +trim_trailing_whitespace = true + +[Dockerfile] +indent_size = 4 + +[*.md] +max_line_length = 0 +trim_trailing_whitespace = false + +[COMMIT_EDITMSG] +max_line_length = 0 diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..db263d7 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,12 @@ +sudo: required + +services: + - docker + +script: + - docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD" + - ./builder.sh + +branches: + only: + - master \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..f11ff2f --- /dev/null +++ b/README.md @@ -0,0 +1,122 @@ +# OpenWRT Docker Builder + +Docker images for building packages and firmware images + +## What does it do? + +OpenWRT Build System is really huge, and takes a lot of time to build up +things, so in order to make it easier to other Devs to use it, the Build +System outputs two other build systems: the OpenWRT Package Builder and the +OpenWRT Image/Firmware Builder. + +### OpenWRT Package Builder + +The Package Builder allows developers to build up a specific set of packages +without having to build the entire packaging system. So this means that the +developers just need to plug-in their Package Feeds and set which packages +are meant to be built. + +### OpenWRT Image/Firmware Builder + +The Image Builder allows developers to build an Image/Firmware to deploy on +their devices without having to build all the packages from scratch. +This allows Continuous Delivery systems to be much more eficients since it is +just installing packages into newly file-systems. + +## Why? + +I've build this because of disadvantages that I found on not having this: + +1. Setup - It takes some time to setup. And on Docker-based CI environments, +in case things aren't cleared properly, it introduces errors in between of +build iterations. + +2. Disk usage - A full setup for a target really takes a lot of space. It's +better to have some setup where we have ephemeral changes between the build +iterations, but only if we can cache the initial setup. (which we can by +using docker) + +## Architecture + +### Base image +`cusspvz/openwrt-builder:base` + +We have a base image where all the other images are based from. This image is +based on Debian and includes all the needed tools to the OpenWRT Builders. + +### OpenWRT Package Builder +`cusspvz/openwrt-builder:package-builder_VERSION_TARGET[_SUBTARGET]` + +Examples: `cusspvz/openwrt-builder:package-builder_18.0.1_brcm2708-bcm2710` +Examples: `cusspvz/openwrt-builder:package-builder_17.01.6_brcm2708-bcm2710` + + +### OpenWRT Image/Firmware Builder +`cusspvz/openwrt-builder:image-builder_VERSION_TARGET[_SUBTARGET]` + +Examples: `cusspvz/openwrt-builder:image-builder_18.0.1_brcm2708-bcm2710` +Examples: `cusspvz/openwrt-builder:image-builder_17.01.6_brcm2708-bcm2710` + +#### Docker Container folders: +`/src` - Builder Source +`/feeds` - folder to link custom feeds. The image detects mounted folders +automatically, so there's no need to tell us which feeds you want to build. +`/overlay` - folder with files and folders to overlay on the images root +`/output` - folder to output built images + +## Usage + +### OpenWRT Package Builder + +Builds `.opkg` files and a `Packages.gz` + +```bash + +docker run -ti --rm \ + -e PACKAGES="transmission openvpn node node-npm" \ + -v /path/to/custom-packages-feed:/feeds/mypackages:z \ + -v /path/to/output-dir:/output:z \ + cusspvz/openwrt-builder:package-builder_18.0.1_brcm2708-bcm2710 + +``` + +### OpenWRT Image/Firmware Builder + +Builds all the target's images. + +```bash + +docker run -ti --rm \ + -e PACKAGES="-luci transmission openvpn node node-npm" \ + -e CUSTOM_FEEDS="mypackages" \ + -v /path/to/custom-packages-feed:/feeds/mypackages:z \ + -v /path/to/overlay-dir:/overlay:z \ + -v /path/to/output-dir:/output:z \ + cusspvz/openwrt-builder:image-builder_18.0.1_brcm2708-bcm2710 + +``` + + +## Donate + +Want to buy me a cup of coffee? + +BTC: 3FyTUneEqXrpRyCjmXvH4kdmvg7Tomwc4j +LTC: MFyux9RBvgjy79iQDgtegYMkJbUqiC27i7 +ETH: 0xa2b5Be27d03916E48Ae445A48d784B0E3cBD825a +ETC: 0xC4b531135a381d2A91F718249eb33a90f187B231 +BTC-CASH: qpyme0065wt6c37jart8dn7fy8ldwua74spr2s7vy0 + +Thanks + + +## Credits + +Thanks to the user [jandelgado](https://github.com/jandelgado) for his work on the +docker builder and the docker compiler. + +This exists thanks to: +- @OpenWRT team on [OpenWRT](https://github.com/openwrt/openwrt) +- @jandelgado on [lede-dockercompiler](https://github.com/jandelgado/lede-dockercompiler) +- @jandelgado on [lede-dockerbuilder](https://github.com/jandelgado/lede-dockerbuilder) + diff --git a/base-image/Dockerfile b/base-image/Dockerfile new file mode 100644 index 0000000..02bb3ea --- /dev/null +++ b/base-image/Dockerfile @@ -0,0 +1,24 @@ +FROM debian:8 +LABEL MAINTAINER "Jose Moreira " + +RUN echo "deb http://ftp.debian.org/debian jessie-backports main" >> /etc/apt/sources.list \ + && apt-get update \ + && apt-get -t jessie-backports install "gosu" \ + && apt-get install -y --no-install-recommends \ + ca-certificates wget curl rsync \ + git subversion \ + build-essential g++ \ + python time unzip file gawk ccache gettext xsltproc \ + libssl-dev libncurses5-dev zlib1g-dev \ + && apt-get autoclean \ + && apt-get clean \ + && apt-get autoremove \ + && rm -rf /var/lib/apt/lists/* + +ADD etc/entrypoint.sh /entrypoint +RUN chmod 755 /entrypoint + +RUN mkdir -p /src /output +WORKDIR /src +ENTRYPOINT ["/entrypoint"] +CMD ["/bin/bash"] \ No newline at end of file diff --git a/base-image/entrypoint.sh b/base-image/entrypoint.sh new file mode 100644 index 0000000..db18de4 --- /dev/null +++ b/base-image/entrypoint.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +chown -R $GOSU_USER /lede + +# If GOSU_USER environment variable set to something other than 0:0 (root:root), +# become user:group set within and exec command passed in args +if [ "$GOSU_USER" != "0:0" ]; then + # make sure a valid user exists in /etc/passwd + if grep "^builder:" /etc/passwd; then + sed -i "/^builder:/d" /etc/passwd + fi + echo "builder:x:$GOSU_USER:LEDE builder:/lede:/bin/bash" >> /etc/passwd + exec gosu $GOSU_USER "$@" +fi + +# If GOSU_USER was 0:0 exec command passed in args without gosu (assume already root) +exec "$@" diff --git a/builder.sh b/builder.sh new file mode 100755 index 0000000..1dd4d56 --- /dev/null +++ b/builder.sh @@ -0,0 +1,87 @@ +#!/bin/bash + +DOCKER_USE_SUDO=${DOCKER_USE_SUDO:-0} +DOCKER_USERNAME=${DOCKER_USERNAME:-cusspvz} +DOCKER_PASSWORD=${DOCKER_PASSWORD} +DOCKER_IMAGE=${DOCKER_IMAGE:-$DOCKER_USERNAME/openwrt-builder} + +DOCKER="${DOCKER:-docker}" + +CLOSE_EXEC="/dev/null" +# CLOSE_EXEC=">/dev/null 2>&1" +if [ ! -z DEBUG ]; then + CLOSE_EXEC="/dev/null" +fi + +function safeexit () { + echo "$1" + [ ! -z $2 ] && exit $2 || exit 0 +} + +if [ -z $DOCKER_USERNAME ] || [ -z $DOCKER_PASSWORD ]; then + safeexit "Please make sure that you've specified the DOCKER_USERNAME and DOCKER_PASSWORD envs" 1 +fi + + +if [ "$DOCKER_USE_SUDO" != "0" ]; then + DOCKER="sudo ${DOCKER}" +fi + +function docker_tag_exists() { + curl --silent -f -lSL https://index.docker.io/v1/repositories/$1/tags/$2 &>$CLOSE_EXEC; +} + +function generate_dockerfile_based_on() { + echo "FROM $1:base" + cat $2 +} + +# MAIN - this is where it starts + +echo "BASE - Building/Fetching base image" + +# Lets pull up the base image +$DOCKER pull "$DOCKER_IMAGE:base" &>$CLOSE_EXEC; + +# Now that we have it in cache, lets build the base image to ensure +# that it has the same output +$DOCKER build -t "$DOCKER_IMAGE:base" ./base-image &>$CLOSE_EXEC; + +# Push the docker base in case it gets changed +$DOCKER push "$DOCKER_IMAGE:base" &>$CLOSE_EXEC; + +# Now it's time to do the same +for VERSION in $(ls targets/); do +for TARGET in $(ls targets/${VERSION}/); do + echo "${VERSION} ${TARGET} - Loading up target configs" + eval "$(cat targets/${VERSION}/${TARGET})" + + DOCKER_PACKAGE_BUILDER_TAG="package-builder_${VERSION}_${TARGET}" + DOCKER_IMAGE_BUILDER_TAG="image-builder_${VERSION}_${TARGET}" + + # Handle Package builder + if docker_tag_exists "$DOCKER_IMAGE" "$DOCKER_PACKAGE_BUILDER_TAG"; then + echo "${VERSION} ${TARGET} - Package Builder already exists" + else + echo "${VERSION} ${TARGET} - Building Package Builder ..." + generate_dockerfile_based_on $DOCKER_IMAGE ./package-builder/Dockerfile | $DOCKER build -f - -t "$DOCKER_IMAGE:$DOCKER_PACKAGE_BUILDER_TAG" ./package-builder &>$CLOSE_EXEC; + + echo "${VERSION} ${TARGET} - Pushing Package Builder ..." + $DOCKER push "$DOCKER_IMAGE:$DOCKER_PACKAGE_BUILDER_TAG" &>$CLOSE_EXEC; + fi + + # Handle Image builder + if docker_tag_exists "$DOCKER_IMAGE" "$DOCKER_IMAGE_BUILDER_TAG"; then + echo "${VERSION} ${TARGET} - Image Builder already exists" + else + echo "${VERSION} ${TARGET} - Building Image Builder ..." + generate_dockerfile_based_on $DOCKER_IMAGE ./image-builder/Dockerfile | $DOCKER build -f - -t "$DOCKER_IMAGE:$DOCKER_IMAGE_BUILDER_TAG" ./image-builder &>$CLOSE_EXEC; + + echo "${VERSION} ${TARGET} - Pushing Image Builder ..." + $DOCKER push "$DOCKER_IMAGE:$DOCKER_IMAGE_BUILDER_TAG" &>$CLOSE_EXEC; + fi + +done; +done; + +echo "Builder is now going to rest ..." diff --git a/image-builder/Dockerfile b/image-builder/Dockerfile new file mode 100644 index 0000000..e69de29 diff --git a/image-builder/builder.sh b/image-builder/builder.sh new file mode 100644 index 0000000..a9bf588 --- /dev/null +++ b/image-builder/builder.sh @@ -0,0 +1 @@ +#!/bin/bash diff --git a/package-builder/Dockerfile b/package-builder/Dockerfile new file mode 100644 index 0000000..6aae00c --- /dev/null +++ b/package-builder/Dockerfile @@ -0,0 +1,2 @@ +LABEL MAINTAINER "Jose Moreira " + diff --git a/package-builder/builder.sh b/package-builder/builder.sh new file mode 100644 index 0000000..1635256 --- /dev/null +++ b/package-builder/builder.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +PATH_SRC=/src +PATH_FEEDS=/feeds +CUSTOM_FEEDS=$(ls $PATH_FEEDS) +CPUS=${CPUS:-2} +CLEAN=${CLEAN:-0} + +## Verify if we have packages to build +[ -z "$PACKAGES" ] && { + echo "Please provide a list of packages for us to build" + exit 1 +} + +## HANDLE FEEDS +cp $PATH_SRC/feeds.conf.default $PATH_SRC/feeds.conf +for CUSTOM_FEED in $CUSTOM_FEEDS; do + echo "src-link ${CUSTOM_FEED} ${PATH_FEEDS}/${CUSTOM_FEED}" >> $PATH_SRC/feeds.conf +done; + +./scripts/feeds update -a +for CUSTOM_FEED in $CUSTOM_FEEDS; do + ./scripts/feeds install -a -p $CUSTOM_FEED +done; + +make defconfig + +for PACKAGE in $PACKAGES; do + [ "$CLEAN" != "0" ] && { + make package/${PACKAGE}/clean + } + + make package/${PACKAGE}/compile +done diff --git a/targets/18.0.1/brcm2708-brcm2710 b/targets/18.0.1/brcm2708-brcm2710 new file mode 100644 index 0000000..f6e1d5d --- /dev/null +++ b/targets/18.0.1/brcm2708-brcm2710 @@ -0,0 +1,2 @@ +OWRT_IMAGE_BUILDER="http://downloads.openwrt.org/releases/18.06.1/targets/brcm2708/bcm2710/openwrt-imagebuilder-18.06.1-brcm2708-bcm2710.Linux-x86_64.tar.xz" +OWRT_SDK="http://downloads.openwrt.org/releases/18.06.1/targets/brcm2708/bcm2710/openwrt-sdk-18.06.1-brcm2708-bcm2710_gcc-7.3.0_musl.Linux-x86_64.tar.xz" \ No newline at end of file