From 4272ad4fd81509ecf767666a2b7968121c108ba6 Mon Sep 17 00:00:00 2001 From: Thorsten Klein Date: Wed, 17 Jun 2020 19:35:42 +0200 Subject: [PATCH] [Enhancement] Improved CI with Drone (#278) * Misc: cleanup and restructure Makefile * Misc: use dind output target in Dockerfile * tests/e2e: ensure that the docker runner is really up before continuing * CI: add drone * Misc: fix image tag for e2e in Makefile * CI/drone: add build step, step conditionals and restrict to linux platform * CI/drone: add docs pipeline * Misc: allow setting e2e runner timeout via Makefile arg * CI/drone: add target platform for docs pipeline * CI: Goodbye travis * Misc: add drone make target to run drone CI locally * CI/drone: add basic pipelines for k3d-tools and k3d-proxy * remove ref event trigger --- .drone.yml | 184 ++++++++++++++++++++++++++++++++++++++++++++++++ .travis.yml | 47 ------------- Dockerfile | 6 +- Makefile | 115 ++++++++++++++++++++++-------- tests/dind.sh | 27 +++++-- tests/runner.sh | 2 +- 6 files changed, 293 insertions(+), 88 deletions(-) create mode 100644 .drone.yml delete mode 100644 .travis.yml diff --git a/.drone.yml b/.drone.yml new file mode 100644 index 00000000..dce6822e --- /dev/null +++ b/.drone.yml @@ -0,0 +1,184 @@ +--- +kind: pipeline +type: docker +name: main + +platform: + os: linux + arch: amd64 + +steps: + + - name: lint + image: golang:1.14 + commands: + - make ci-setup + - make check-fmt lint + when: + event: + - push + - pull_request + - tag + + - name: test + image: docker:19.03 + volumes: + - name: dockersock + path: /var/run + commands: + - apk add git bash curl sudo jq make + - sleep 5 # give docker enough time to start + - make e2e + when: + event: + - push + - pull_request + - tag + + - name: build + image: golang:1.14 + commands: + - make ci-setup + - make build-cross + depends_on: + - lint + - test + when: + branch: + - master + event: + - push + - tag + + - name: release + image: plugins/github-release + settings: + api_key: + from_secret: github_token + files: + - _dist/* + checksum: + - sha256 + depends_on: + - lint + - test + - build + when: + event: + - tag + +services: + # Starting the docker service to be used by dind + - name: docker + image: docker:19.03-dind + privileged: true + volumes: + - name: dockersock + path: /var/run + +volumes: + - name: dockersock + temp: {} + +--- + +kind: pipeline +type: docker +name: docs + +platform: + os: linux + arch: amd64 + +steps: + + - name: build + image: python:3.8 + commands: + - python3 -m pip install -r docs/requirements.txt + - mkdocs build --verbose --clean --strict + when: + branch: + - master + event: + - push + + - name: publish + image: plugins/gh-pages + settings: + password: + from_secret: github_token + username: + from_secret: github_username + pages_directory: site/ + target_branch: gh-pages + when: + branch: + - master + event: + - push + +trigger: + event: + - push + ref: + - master + +--- + +kind: pipeline +type: docker +name: proxy + +platform: + os: linux + arch: amd64 + +steps: + # TODO: use tag from git (without 'proxy-' prefix) + - name: build_push + image: plugins/docker + settings: + repo: rancher/k3d-proxy + dockerfile: proxy/Dockerfile + context: proxy/ + username: + from_secret: dockerhub_username + password: + from_secret: dockerhub_password + when: + event: + - tag + +trigger: + event: + - tag + +--- + +kind: pipeline +type: docker +name: tools + +platform: + os: linux + arch: amd64 + +steps: + - name: build_push + image: plugins/docker + settings: + repo: rancher/k3d-tools + dockerfile: tools/Dockerfile + context: tools/ + username: + from_secret: dockerhub_username + password: + from_secret: dockerhub_password + when: + event: + - tag + +trigger: + event: + - tag diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 3f7806ed..00000000 --- a/.travis.yml +++ /dev/null @@ -1,47 +0,0 @@ -matrix: - include: - - dist: bionic - language: go - env: - - GO111MODULE=on - go: - - 1.14.x - git: - depth: 1 - install: true - before_script: - - make ci-setup - script: - - make -e "E2E_SKIP=test_multi_master" ci-tests build-cross - deploy: - provider: releases - skip_cleanup: true - api_key: - secure: VFb0jmL6t+cuKWAGC5OLeseTJrK3CakD5Laeyp0JVG/o3/cYgHP0lIeDmJJMZL/Luxm5aL8QHsCRfQoMBAnj6Q8hchI9rbcYhFZzuFJYyDxjcrPm0+kM3yiK14lQZNvksy2ZSsO7D63c8x9sQVrEus4idVTUoxrpSXLM2eVjl6W0O2RdZvLsxgaLPwV1ufpihrqbXdEUjt/YSYpHiC5gS3o+FcyMGucJQdN/L7p6jyAqVgg4+t8bdyWj6+MEG4p8lmWhhbGzDo38iMxtCBu+nDHRsbivay3eJZ643VguX0lj62Vt5KUTcVJntmZqQ2UF6FoEVUPOegkrSeoiMuOH1+nYwcsfMFijMkrcFhb6bAisJJd6agdhFWXiSwL88FQkJh0DqeA0tFFIzDbTS/AZTY4Li8bWng3aCBgSXiMzIBf0es+wMDw0gwhfH44Y/RAsKSQJ/Lln00AaVzkOkOWOmu5Ks0CVYDy0M5QDQOCW2E9TIb7WdIMh3aNCkZi+rGovigejJv3vUZqkN03Og07Hbrjgfg28iY3isIt3soOrVqek2hJJFnKjUhhv2OhJm3z6FpTyMViUtSmJ+LTiBjpyiWC4QuaITDadCJTxZQwobhI+18c2Zi5/HjTX1pgD1wk3quv9R4bGjVINenefG6xxaNj+CeFTfrQnnHuXOL50828= - file: - - _dist/k3d-darwin-amd64 - - _dist/k3d-linux-386 - - _dist/k3d-linux-amd64 - - _dist/k3d-linux-arm - - _dist/k3d-linux-arm64 - - _dist/k3d-windows-amd64.exe - on: - repo: rancher/k3d - tags: true - - dist: bionic - language: python - python: 3.8 - before_script: - - python3 -m pip install -r docs/requirements.txt - script: - - mkdocs build --verbose --clean --strict - deploy: - provider: pages - skip_cleanup: true - github_token: - secure: VFb0jmL6t+cuKWAGC5OLeseTJrK3CakD5Laeyp0JVG/o3/cYgHP0lIeDmJJMZL/Luxm5aL8QHsCRfQoMBAnj6Q8hchI9rbcYhFZzuFJYyDxjcrPm0+kM3yiK14lQZNvksy2ZSsO7D63c8x9sQVrEus4idVTUoxrpSXLM2eVjl6W0O2RdZvLsxgaLPwV1ufpihrqbXdEUjt/YSYpHiC5gS3o+FcyMGucJQdN/L7p6jyAqVgg4+t8bdyWj6+MEG4p8lmWhhbGzDo38iMxtCBu+nDHRsbivay3eJZ643VguX0lj62Vt5KUTcVJntmZqQ2UF6FoEVUPOegkrSeoiMuOH1+nYwcsfMFijMkrcFhb6bAisJJd6agdhFWXiSwL88FQkJh0DqeA0tFFIzDbTS/AZTY4Li8bWng3aCBgSXiMzIBf0es+wMDw0gwhfH44Y/RAsKSQJ/Lln00AaVzkOkOWOmu5Ks0CVYDy0M5QDQOCW2E9TIb7WdIMh3aNCkZi+rGovigejJv3vUZqkN03Og07Hbrjgfg28iY3isIt3soOrVqek2hJJFnKjUhhv2OhJm3z6FpTyMViUtSmJ+LTiBjpyiWC4QuaITDadCJTxZQwobhI+18c2Zi5/HjTX1pgD1wk3quv9R4bGjVINenefG6xxaNj+CeFTfrQnnHuXOL50828= - local_dir: site - on: - repo: rancher/k3d - branch: master - diff --git a/Dockerfile b/Dockerfile index 8ca68f03..507ae265 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,10 +3,8 @@ WORKDIR /app COPY . . RUN make build && bin/k3d version -FROM docker:19.03-dind - -# TODO: we could create a different stage for e2e tests -RUN apk add bash curl sudo +FROM docker:19.03-dind as dind +RUN apk add bash curl sudo jq RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl && \ chmod +x ./kubectl && \ mv ./kubectl /usr/local/bin/kubectl diff --git a/Makefile b/Makefile index 8eea3767..29a81976 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,19 @@ +################################### +# # +# CONFIGURATION # +# # +################################### + +########## Shell/Terminal Settings ########## SHELL := /bin/bash -# Build targets -TARGETS ?= darwin/amd64 linux/amd64 linux/386 linux/arm linux/arm64 windows/amd64 -TARGET_OBJS ?= darwin-amd64.tar.gz darwin-amd64.tar.gz.sha256 linux-amd64.tar.gz linux-amd64.tar.gz.sha256 linux-386.tar.gz linux-386.tar.gz.sha256 linux-arm.tar.gz linux-arm.tar.gz.sha256 linux-arm64.tar.gz linux-arm64.tar.gz.sha256 windows-amd64.zip windows-amd64.zip.sha256 +# determine if make is being executed from interactive terminal +INTERACTIVE:=$(shell [ -t 0 ] && echo 1) + +# Use Go Modules for everything +export GO111MODULE=on + +########## Tags ########## # get git tag GIT_TAG := $(shell git describe --tags) @@ -10,6 +21,9 @@ ifeq ($(GIT_TAG),) GIT_TAG := $(shell git describe --always) endif +# Docker image tag derived from Git tag +K3D_IMAGE_TAG := $(GIT_TAG) + # get latest k3s version: grep the tag and replace + with - (difference between git and dockerhub tags) K3S_TAG := $(shell curl --silent "https://update.k3s.io/v1-release/channels/stable" | egrep -o '/v[^ ]+"' | sed -E 's/\/|\"//g' | sed -E 's/\+/\-/') @@ -19,12 +33,21 @@ $(warning Output of curl: $(shell curl --silent "https://update.k3s.io/v1-releas $(error exiting) endif -# determine if make is being executed from interactive terminal -INTERACTIVE:=$(shell [ -t 0 ] && echo 1) +########## Source Options ########## +# DIRS defines a single level directly, we only look at *.go in this directory. +# REC_DIRS defines a source code tree. All go files are analyzed recursively. +DIRS := . +REC_DIRS := cmd -# E2E test settings +########## Test Settings ########## E2E_LOG_LEVEL ?= WARN E2E_SKIP ?= +E2E_RUNNER_START_TIMEOUT ?= 10 + +########## Go Build Options ########## +# Build targets +TARGETS ?= darwin/amd64 linux/amd64 linux/386 linux/arm linux/arm64 windows/amd64 +TARGET_OBJS ?= darwin-amd64.tar.gz darwin-amd64.tar.gz.sha256 linux-amd64.tar.gz linux-amd64.tar.gz.sha256 linux-386.tar.gz linux-386.tar.gz.sha256 linux-arm.tar.gz linux-arm.tar.gz.sha256 linux-arm64.tar.gz linux-arm64.tar.gz.sha256 windows-amd64.zip windows-amd64.zip.sha256 # Go options GO ?= go @@ -38,50 +61,59 @@ GOFLAGS := BINDIR := $(CURDIR)/bin BINARIES := k3d -K3D_IMAGE_TAG := $(GIT_TAG) +# Rules for finding all go source files using 'DIRS' and 'REC_DIRS' +GO_SRC := $(foreach dir,$(DIRS),$(wildcard $(dir)/*.go)) +GO_SRC += $(foreach dir,$(REC_DIRS),$(shell find $(dir) -name "*.go")) +########## Required Tools ########## # Go Package required PKG_GOX := github.com/mitchellh/gox@v1.0.1 PKG_GOLANGCI_LINT_VERSION := 1.25.0 PKG_GOLANGCI_LINT_SCRIPT := https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh PKG_GOLANGCI_LINT := github.com/golangci/golangci-lint/cmd/golangci-lint@v${PKG_GOLANGCI_LINT_VERSION} +########## Linting Options ########## # configuration adjustments for golangci-lint GOLANGCI_LINT_DISABLED_LINTERS := "" # disabling typecheck, because it currently (06.09.2019) fails with Go 1.13 -# Use Go Modules for everything -export GO111MODULE=on - -# go source directories. -# DIRS defines a single level directly, we only look at *.go in this directory. -# REC_DIRS defines a source code tree. All go files are analyzed recursively. -DIRS := . -REC_DIRS := cmd - -# Rules for finding all go source files using 'DIRS' and 'REC_DIRS' -GO_SRC := $(foreach dir,$(DIRS),$(wildcard $(dir)/*.go)) -GO_SRC += $(foreach dir,$(REC_DIRS),$(shell find $(dir) -name "*.go")) - # Rules for directory list as input for the golangci-lint program LINT_DIRS := $(DIRS) $(foreach dir,$(REC_DIRS),$(dir)/...) +############################# +# # +# TARGETS # +# # +############################# + .PHONY: all build build-cross clean fmt check-fmt lint check extra-clean install-tools all: clean fmt check build +############################ +########## Builds ########## +############################ + +# debug builds build-debug: GCFLAGS+="all=-N -l" build-debug: build +# default build target for the local platform build: CGO_ENABLED=0 $(GO) build $(GOFLAGS) -tags '$(TAGS)' -ldflags '$(LDFLAGS)' -gcflags '$(GCFLAGS)' -o '$(BINDIR)/$(BINARIES)' +# cross-compilation for all targets build-cross: LDFLAGS += -extldflags "-static" build-cross: CGO_ENABLED=0 gox -parallel=3 -output="_dist/$(BINARIES)-{{.OS}}-{{.Arch}}" -osarch='$(TARGETS)' $(GOFLAGS) $(if $(TAGS),-tags '$(TAGS)',) -ldflags '$(LDFLAGS)' -build-docker: Dockerfile - @echo "Building Docker image k3d:$(K3D_IMAGE_TAG)" - docker build -t k3d:$(K3D_IMAGE_TAG) . +# build a specific docker target ( '%' matches the target as specified in the Dockerfile) +build-docker-%: + @echo "Building Docker image k3d:$(K3D_IMAGE_TAG)-$*" + docker build . -t k3d:$(K3D_IMAGE_TAG)-$* --target $* + +############################## +########## Cleaning ########## +############################## clean: @rm -rf $(BINDIR) _dist/ @@ -90,13 +122,14 @@ extra-clean: clean $(GO) clean -i $(PKG_GOX) $(GO) clean -i $(PKG_GOLANGCI_LINT) +########################################## +########## Formatting & Linting ########## +########################################## + # fmt will fix the golang source style in place. fmt: @gofmt -s -l -w $(GO_SRC) -e2e: build-docker - @echo "Running e2e tests in k3d:$(K3D_IMAGE_TAG)" - LOG_LEVEL="$(E2E_LOG_LEVEL)" E2E_SKIP="$(E2E_SKIP)" tests/dind.sh "${K3D_IMAGE_TAG}" # check-fmt returns an error code if any source code contains format error. check-fmt: @test -z $(shell gofmt -s -l $(GO_SRC) | tee /dev/stderr) || echo "[WARN] Fix formatting issues with 'make fmt'" @@ -106,6 +139,28 @@ lint: check: check-fmt lint +########################### +########## Tests ########## +########################### + +e2e: build-docker-dind + @echo "Running e2e tests in k3d:$(K3D_IMAGE_TAG)" + LOG_LEVEL="$(E2E_LOG_LEVEL)" E2E_SKIP="$(E2E_SKIP)" E2E_RUNNER_START_TIMEOUT=$(E2E_RUNNER_START_TIMEOUT) tests/dind.sh "${K3D_IMAGE_TAG}-dind" + +ci-tests: fmt check e2e + +########################## +########## Misc ########## +########################## + +drone: + @echo "Running drone pipeline locally with branch=master and event=push" + drone exec --trusted --branch master --event push + +######################################### +########## Setup & Preparation ########## +######################################### + # Check for required executables HAS_GOX := $(shell command -v gox 2> /dev/null) HAS_GOLANGCI := $(shell command -v golangci-lint) @@ -129,6 +184,10 @@ endif endif endif +# In the CI system, we need... +# - golangci-lint for linting (lint) +# - gox for cross-compilation (build-cross) +# - kubectl for E2E-tests (e2e) ci-setup: @echo "Installing Go tools..." curl -sfL $(PKG_GOLANGCI_LINT_SCRIPT) | sh -s -- -b ${GOPATH}/bin v$(PKG_GOLANGCI_LINT_VERSION) @@ -137,6 +196,4 @@ ci-setup: @echo "Installing kubectl..." curl -LO https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl chmod +x ./kubectl - sudo mv ./kubectl /usr/local/bin/kubectl - -ci-tests: fmt check e2e + mv ./kubectl /usr/local/bin/kubectl diff --git a/tests/dind.sh b/tests/dind.sh index bc41b829..58aa6185 100755 --- a/tests/dind.sh +++ b/tests/dind.sh @@ -6,23 +6,24 @@ K3D_IMAGE_TAG=$1 # define E2E_KEEP to non-empty for keeping the e2e runner container after running the tests E2E_KEEP=${E2E_KEEP:-} -#################################################################################### +# Max. time to wait for the runner container to be up +RUNNER_START_TIMEOUT=${E2E_RUNNER_START_TIMEOUT:-10} -TIMESTAMP=$(date "+%m%d%H%M%S") +#################################################################################### +# Start the runner container +TIMESTAMP=$(date "+%y%m%d%H%M%S") k3de2e=$(docker run -d \ - -v "$(pwd)"/tests:/tests \ + -v "$(pwd)/tests:/tests" \ --privileged \ -e EXE="$K3D_EXE" \ -e CI="true" \ -e LOG_LEVEL="$LOG_LEVEL" \ -e E2E_SKIP="$E2E_SKIP" \ --name "k3d-e2e-runner-$TIMESTAMP" \ - k3d:$K3D_IMAGE_TAG) - -sleep 5 # wait 5 seconds for docker to start + "k3d:$K3D_IMAGE_TAG") -# Execute tests +# setup exit trap (make sure that we always stop and remove the runner container) finish() { docker stop "$k3de2e" || /bin/true if [ -z "$E2E_KEEP" ] ; then @@ -31,4 +32,16 @@ finish() { } trap finish EXIT +# wait for the runner container to be up or exit early +TIMEOUT=0 +until docker inspect "$k3de2e" | jq ".[0].State.Running" && docker logs "$k3de2e" 2>&1 | grep -i "API listen on /var/run/docker.sock"; do + if [[ $TIMEOUT -eq $RUNNER_START_TIMEOUT ]]; then + echo "Failed to start E2E Runner Container in $RUNNER_START_TIMEOUT seconds" + exit 1 + fi + sleep 1 + (( TIMEOUT++ )) +done + +# execute tests docker exec "$k3de2e" /tests/runner.sh diff --git a/tests/runner.sh b/tests/runner.sh index 7a889529..16ebf1ee 100755 --- a/tests/runner.sh +++ b/tests/runner.sh @@ -14,7 +14,7 @@ source "$CURR_DIR/common.sh" info "Preparing filesystem and environment..." -mkdir -p /root/.kube +mkdir -p $HOME/.kube for i in $CURR_DIR/test_*.sh ; do base=$(basename $i .sh)