Core/grin
0
0
Fork 0
mirror of https://github.com/mimblewimble/grin.git synced 2025-01-20 19:11:08 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

refactor(ci): reorganize CI related code (#2658)

Browse source 
Break-down the CI related code into smaller more maintainable pieces.
This commit is contained in:
Peter Mrekaj 2019-03-11 16:23:16 +01:00 committed by Gary Yu
parent a7a160a5d4
commit c0880443c0
5 changed files with 192 additions and 112 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

69
.auto-release.sh
View file

@ -1,69 +0,0 @@
#!/bin/bash
repo_slug="mimblewimble/grin"
token="$GITHUB_TOKEN"
export CHANGELOG_GITHUB_TOKEN="$token"
tagname=`git describe --tags --exact-match 2>/dev/null || git symbolic-ref -q --short HEAD`
echo 'make a tarball for the release binary...\n'
if [[ $TRAVIS_OS_NAME == 'osx' ]]; then
# Do some custom requirements on OS X
cd target/release ; rm -f *.tgz; tar zcf "grin-$tagname-$TRAVIS_JOB_ID-osx.tgz" grin
/bin/ls -ls *.tgz | awk '{print $6,$7,$8,$9,$10}'
md5 "grin-$tagname-$TRAVIS_JOB_ID-osx.tgz" > "grin-$tagname-$TRAVIS_JOB_ID-osx.tgz"-md5sum.txt
/bin/ls -ls *-md5sum.txt | awk '{print $6,$7,$8,$9,$10}'
cd - > /dev/null;
echo "tarball generated\n"
# Only generate changelog on Linux platform, to avoid duplication
exit 0
else
# Do some custom requirements on Linux
cd target/release ; rm -f *.tgz; tar zcf "grin-$tagname-$TRAVIS_JOB_ID-linux-amd64.tgz" grin
/bin/ls -ls *.tgz | awk '{print $6,$7,$8,$9,$10}'
md5sum "grin-$tagname-$TRAVIS_JOB_ID-linux-amd64.tgz" > "grin-$tagname-$TRAVIS_JOB_ID-linux-amd64.tgz"-md5sum.txt
/bin/ls -ls *-md5sum.txt | awk '{print $6,$7,$8,$9,$10}'
cd - > /dev/null;
echo "tarball generated\n"
fi
version="$tagname"
branch="`git symbolic-ref -q --short HEAD`"
# automatic changelog generator
gem install github_changelog_generator
LAST_REVISION=$(git rev-list --tags --skip=1 --max-count=1)
LAST_RELEASE_TAG=$(git describe --abbrev=0 --tags ${LAST_REVISION})
# Generate CHANGELOG.md
github_changelog_generator \
-u $(cut -d "/" -f1 <<< $repo_slug) \
-p $(cut -d "/" -f2 <<< $repo_slug) \
--since-tag ${LAST_RELEASE_TAG}
body="$(cat CHANGELOG.md)"
# Overwrite CHANGELOG.md with JSON data for GitHub API
jq -n \
--arg body "$body" \
--arg name "$version" \
--arg tag_name "$version" \
--arg target_commitish "$branch" \
'{
body: $body,
name: $name,
tag_name: $tag_name,
target_commitish: $target_commitish,
draft: false,
prerelease: false
}' > CHANGELOG.md
release_id="$(curl -0 -XGET -H "Authorization: token $token" https://api.github.com/repos/$repo_slug/releases/tags/$tagname 2>/dev/null | grep id | head -n 1 | sed 's/ *"id": *\(.*\),/\1/')"
echo "Updating release $version for repo: $repo_slug, branch: $branch. release id: $release_id"
curl -H "Authorization: token $token" --request PATCH --data @CHANGELOG.md "https://api.github.com/repos/$repo_slug/releases/$release_id"
echo "auto changelog uploaded.\n"

28
.ci/general-jobs Executable file
View file

@ -0,0 +1,28 @@
#!/usr/bin/env bash
# Copyright 2019 The Grin Developers
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This script contains general jobs.
case "${CI_JOB}" in
"test")
for dir in ${CI_JOB_ARGS}; do
printf "executing tests in directory \`%s\`...\n" "${dir}"
cd "${dir}" && \
cargo test --release && \
cd - > /dev/null || exit 1
done
;;
esac

122
.ci/release-jobs Executable file
View file

@ -0,0 +1,122 @@
#!/usr/bin/env bash
# Copyright 2019 The Grin Developers
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This script contains release-related jobs.
# Redeclare CI and VCP specific environment variables
# to make future migration to other providers easier.
readonly JOB_ID="${TRAVIS_JOB_ID}"
readonly OS_NAME="${TRAVIS_OS_NAME}"
readonly TEST_RESULT="${TRAVIS_TEST_RESULT}"
readonly VCP_AUTH_TOKEN="${GITHUB_TOKEN}"
case "${CI_JOB}" in
"release")
# The release can only be triggered after successful completion of all tests.
[[ "${TEST_RESULT}" != 0 ]] && exit 1
readonly REPO_TAG="$(git describe --tags --exact-match 2> /dev/null || git symbolic-ref -q --short HEAD)"
case "${OS_NAME}" in
"linux")
cargo clean && \
cargo build --release
readonly ARCHIVE_CMD="tar zcf"
readonly BIN_SUFFIX=""
readonly PKG_NAME="grin-${REPO_TAG}-${JOB_ID}-linux-amd64"
readonly PKG_SUFFIX=".tgz"
;;
"osx")
brew update
cargo clean && \
cargo build --release
readonly ARCHIVE_CMD="tar zcf"
readonly BIN_SUFFIX=""
readonly PKG_NAME="grin-${REPO_TAG}-${JOB_ID}-osx"
readonly PKG_SUFFIX=".tgz"
;;
"windows")
cargo clean && \
cargo build --release
readonly ARCHIVE_CMD="7z a -tzip"
readonly BIN_SUFFIX=".exe"
readonly PKG_NAME="grin-${REPO_TAG}-${JOB_ID}-win-x64"
readonly PKG_SUFFIX=".zip"
;;
*)
printf "Error! Unknown \$OS_NAME: \`%s\`" "${OS_NAME}"
exit 1
esac
printf "creating package \`%s\` for the release binary...\n" "${PKG_NAME}${PKG_SUFFIX}"
cd ./target/release/ || exit 1
rm -f -- *"${PKG_SUFFIX}"
${ARCHIVE_CMD} "${PKG_NAME}${PKG_SUFFIX}" "grin${BIN_SUFFIX}"
ls -ls -- *.tgz | cut -d' ' -f6-
openssl md5 "${PKG_NAME}${PKG_SUFFIX}" > "${PKG_NAME}${PKG_SUFFIX}-md5sum.txt"
ls -ls -- *-md5sum.txt | cut -d' ' -f6-
cd - > /dev/null || exit 1
printf "%s package \`%s\` generated\n" "${OS_NAME}" "${PKG_NAME}${PKG_SUFFIX}"
# Generate changelog only on the Linux platform to avoid duplication.
[[ "${OS_NAME}" != "linux" ]] && exit 0
# Generate CHANGELOG.md
readonly REPO_SLUG="mimblewimble/grin"
readonly REPO_BRANCH="$(git symbolic-ref -q --short HEAD)"
readonly REPO_PREV_RELEASE_TAG="$(git describe --abbrev=0 --tags "$(git rev-list --tags --skip=0 --max-count=1)")"
gem install github_changelog_generator
# Needed by github_changelog_generator.
export CHANGELOG_GITHUB_TOKEN="${VCP_AUTH_TOKEN}"
github_changelog_generator \
--user "$(cut -d "/" -f1 <<< ${REPO_SLUG})" \
--project "$(cut -d "/" -f2 <<< ${REPO_SLUG})" \
--since-tag "${REPO_PREV_RELEASE_TAG}"
readonly CHANGELOG_CONTENT="$(<CHANGELOG.md)"
# Overwrite CHANGELOG.md with JSON data for release patch.
jq --null-input \
--arg body "${CHANGELOG_CONTENT}" \
--arg name "${REPO_TAG}" \
--arg tag_name "${REPO_TAG}" \
--arg target_commitish "${REPO_BRANCH}" \
'{
body: $body,
name: $name,
tag_name: $tag_name,
target_commitish: $target_commitish,
draft: false,
prerelease: false
}' > CHANGELOG.md
readonly HEADERS="Authorization: token ${VCP_AUTH_TOKEN}"
readonly RELEASE_URL="https://api.github.com/repos/${REPO_SLUG}/releases"
readonly RELEASE_ID="$(curl -0 --request GET -H "${HEADERS}" "${RELEASE_URL}/tags/${REPO_TAG}" 2> /dev/null | grep id | head -n 1 | sed 's/ *"id": *\(.*\),/\1/')"
printf "updating release changelog %s for repo: %s, branch: %s, release id: %s\n" "${REPO_TAG}" "${REPO_SLUG}" "${REPO_BRANCH}" "${RELEASE_ID}"
curl -H "${HEADERS}" --request PATCH --data @CHANGELOG.md "${RELEASE_URL}/${RELEASE_ID}"
printf "changelog uploaded.\n"
;;
esac

79
.travis.yml
View file

@ -1,24 +1,28 @@
language: rust
git:
depth: false
# Copyright 2019 The Grin Developers
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
dist: trusty
sudo: required
cache:
cargo: true
timeout: 240
directories:
- $HOME/.cargo
- $TRAVIS_BUILD_DIR/target
before_cache:
- rm -rf $TRAVIS_BUILD_DIR/target/tmp
language: rust
rust:
- stable
git:
depth: false
addons:
apt:
sources:
@ -33,6 +37,13 @@ addons:
- gcc
- binutils-dev
cache:
cargo: true
timeout: 240
directories:
- $HOME/.cargo
- $TRAVIS_BUILD_DIR/target
env:
global:
- RUST_BACKTRACE="1"
@ -41,41 +52,29 @@ env:
matrix:
include:
- os: linux
env: TEST_SUITE=servers
env: CI_JOB="test" CI_JOB_ARGS="servers"
- os: linux
env: TEST_SUITE=chain-core
env: CI_JOB="test" CI_JOB_ARGS="chain core"
- os: linux
env: TEST_SUITE=pool-p2p-src
env: CI_JOB="test" CI_JOB_ARGS="pool p2p src"
- os: linux
env: TEST_SUITE=keychain-wallet
env: CI_JOB="test" CI_JOB_ARGS="keychain wallet"
- os: linux
env: TEST_SUITE=api-util-store
env: CI_JOB="test" CI_JOB_ARGS="api util store"
- os: linux
env: CI_JOB="release" CI_JOB_ARGS=
- os: osx
env: TEST_SUITE=release
env: CI_JOB="release" CI_JOB_ARGS=
# - os: windows
# env: CI_JOB="release" CI_JOB_ARGS=
script:
- IFS='-' read -r -a DIRS <<< "$TEST_SUITE"; DIR=${DIRS[0]};
echo "start testing on folder $DIR...";
if [[ -n "$DIR" ]] && [[ "$TRAVIS_OS_NAME" == "linux" ]]; then cd $DIR && cargo test --release && cd - > /dev/null; fi;
- IFS='-' read -r -a DIRS <<< "$TEST_SUITE"; DIR=${DIRS[1]};
if [[ -n "$DIR" ]]; then
echo "start testing on folder $DIR...";
cd $DIR && cargo test --release && cd - > /dev/null;
fi;
- IFS='-' read -r -a DIRS <<< "$TEST_SUITE"; DIR=${DIRS[2]};
if [[ -n "$DIR" ]]; then
echo "start testing on folder $DIR...";
cd $DIR && cargo test --release && cd - > /dev/null;
fi;
script: .ci/general-jobs
before_cache:
- rm -rf $TRAVIS_BUILD_DIR/target/tmp
before_deploy:
- if [[ "$TEST_SUITE" == "pool-p2p-src" ]]; then
cargo clean && cargo build --release && ./.auto-release.sh;
fi
- if [[ "$TEST_SUITE" == "release" ]] && [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
brew update;
cargo clean && cargo build --release && ./.auto-release.sh;
fi
- bash .ci/release-jobs
deploy:
provider: releases

6
doc/release_instruction.md
View file

@ -123,9 +123,9 @@ Remember to replace `0.3.1-pre1` as the real version, and warmly remind the [[Ve
If you're NOT the owner of the github repo, but at least you have to be a committer which has the right to do a release, the following steps are needed to trigger a version release:
1. Go to release page of the repo, click **Draft a new release**, remember to check the branch is what you're working on! set the **Tag version** to the release number (for example: `0.3.1-pre1`), and set anything in **Release title** and **description**, then click **Publish release**. Don't worry the title and description parts because we need delete it in next step.
1. Because github **release** will be auto-created by our `auto-release` building script, we MUST delete the **release** which we just created in previous step! (Unfortunately, there's no way to only create **tag** by web.)
1. Because github **release** will be auto-created by our `release-jobs` building script, we MUST delete the **release** which we just created in previous step! (Unfortunately, there's no way to only create **tag** by web.)
Even normally Travis-CI need tens of minutes to complete building, I suggest you complete step 2 quickly, otherwise the `auto-release` script will fail on error "release already exist".
Even normally Travis-CI need tens of minutes to complete building, I suggest you complete step 2 quickly, otherwise the `release-jobs` script will fail on error "release already exist".
### 2. Travis-CI Building
@ -135,7 +135,7 @@ The release building is just one of the **TEST_DIRS**, named as `none`. So each
So, the point is: the release building job is that one tagged with `TEST_DIR=none`.
Note: `auto-release` script will only be executed on `deploy` stage, and according to Travis-CI, it will be skipped for any **pull-request** trigger, and since we set `tag: true` it will be only executed when triggered by a tag.
Note: `release-jobs` script will only be executed on `deploy` stage, and according to Travis-CI, it will be skipped for any **pull-request** trigger, and since we set `tag: true` it will be only executed when triggered by a tag.
### 3. Check the Release Page