refactor(ci): reorganize CI related code (#2658)

Break-down the CI related code into smaller more maintainable pieces.

.auto-release.sh

@@ -1,69 +0,0 @@
-#!/bin/bash
-
-repo_slug="mimblewimble/grin"
-token="$GITHUB_TOKEN"
-export CHANGELOG_GITHUB_TOKEN="$token"
-
-tagname=`git describe --tags --exact-match 2>/dev/null || git symbolic-ref -q --short HEAD`
-
-echo 'make a tarball for the release binary...\n'
-
-if [[ $TRAVIS_OS_NAME == 'osx' ]]; then
-
-    # Do some custom requirements on OS X
-    cd target/release ; rm -f *.tgz; tar zcf "grin-$tagname-$TRAVIS_JOB_ID-osx.tgz" grin
-    /bin/ls -ls *.tgz  | awk '{print $6,$7,$8,$9,$10}'
-    md5 "grin-$tagname-$TRAVIS_JOB_ID-osx.tgz" > "grin-$tagname-$TRAVIS_JOB_ID-osx.tgz"-md5sum.txt
-    /bin/ls -ls *-md5sum.txt  | awk '{print $6,$7,$8,$9,$10}'
-    cd - > /dev/null;
-    echo "tarball generated\n"
-
-    # Only generate changelog on Linux platform, to avoid duplication
-    exit 0
-else
-    # Do some custom requirements on Linux
-    cd target/release ; rm -f *.tgz; tar zcf "grin-$tagname-$TRAVIS_JOB_ID-linux-amd64.tgz" grin
-    /bin/ls -ls *.tgz  | awk '{print $6,$7,$8,$9,$10}'
-    md5sum "grin-$tagname-$TRAVIS_JOB_ID-linux-amd64.tgz" > "grin-$tagname-$TRAVIS_JOB_ID-linux-amd64.tgz"-md5sum.txt
-    /bin/ls -ls *-md5sum.txt  | awk '{print $6,$7,$8,$9,$10}'
-    cd - > /dev/null;
-    echo "tarball generated\n"
-fi
-
-version="$tagname"
-branch="`git symbolic-ref -q --short HEAD`"
-
-# automatic changelog generator
-gem install github_changelog_generator
-
-LAST_REVISION=$(git rev-list --tags --skip=1 --max-count=1)
-LAST_RELEASE_TAG=$(git describe --abbrev=0 --tags ${LAST_REVISION})
-
-# Generate CHANGELOG.md
-github_changelog_generator \
-  -u $(cut -d "/" -f1 <<< $repo_slug) \
-  -p $(cut -d "/" -f2 <<< $repo_slug) \
-  --since-tag ${LAST_RELEASE_TAG}
-
-body="$(cat CHANGELOG.md)"
-
-# Overwrite CHANGELOG.md with JSON data for GitHub API
-jq -n \
-  --arg body "$body" \
-  --arg name "$version" \
-  --arg tag_name "$version" \
-  --arg target_commitish "$branch" \
-  '{
-    body: $body,
-    name: $name,
-    tag_name: $tag_name,
-    target_commitish: $target_commitish,
-    draft: false,
-    prerelease: false
-  }' > CHANGELOG.md
-
-release_id="$(curl -0 -XGET -H "Authorization: token $token" https://api.github.com/repos/$repo_slug/releases/tags/$tagname 2>/dev/null | grep id | head -n 1 | sed 's/ *"id": *\(.*\),/\1/')"
-echo "Updating release $version for repo: $repo_slug, branch: $branch. release id: $release_id"
-curl -H "Authorization: token $token" --request PATCH  --data @CHANGELOG.md "https://api.github.com/repos/$repo_slug/releases/$release_id"
-echo "auto changelog uploaded.\n"
-

.ci/general-jobs

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+# Copyright 2019 The Grin Developers
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script contains general jobs.
+
+case "${CI_JOB}" in
+    "test")
+        for dir in ${CI_JOB_ARGS}; do
+            printf "executing tests in directory \`%s\`...\n" "${dir}"
+            cd "${dir}" && \
+            cargo test --release && \
+            cd - > /dev/null || exit 1
+        done
+        ;;
+esac

.ci/release-jobs

@@ -0,0 +1,122 @@
+#!/usr/bin/env bash
+
+# Copyright 2019 The Grin Developers
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script contains release-related jobs.
+
+# Redeclare CI and VCP specific environment variables
+# to make future migration to other providers easier.
+readonly JOB_ID="${TRAVIS_JOB_ID}"
+readonly OS_NAME="${TRAVIS_OS_NAME}"
+readonly TEST_RESULT="${TRAVIS_TEST_RESULT}"
+readonly VCP_AUTH_TOKEN="${GITHUB_TOKEN}"
+
+case "${CI_JOB}" in
+    "release")
+        # The release can only be triggered after successful completion of all tests.
+        [[ "${TEST_RESULT}" != 0 ]] && exit 1
+
+        readonly REPO_TAG="$(git describe --tags --exact-match 2> /dev/null || git symbolic-ref -q --short HEAD)"
+
+        case "${OS_NAME}" in
+            "linux")
+                cargo clean && \
+                cargo build --release
+                readonly ARCHIVE_CMD="tar zcf"
+                readonly BIN_SUFFIX=""
+                readonly PKG_NAME="grin-${REPO_TAG}-${JOB_ID}-linux-amd64"
+                readonly PKG_SUFFIX=".tgz"
+                ;;
+
+            "osx")
+                brew update
+                cargo clean && \
+                cargo build --release
+                readonly ARCHIVE_CMD="tar zcf"
+                readonly BIN_SUFFIX=""
+                readonly PKG_NAME="grin-${REPO_TAG}-${JOB_ID}-osx"
+                readonly PKG_SUFFIX=".tgz"
+                ;;
+
+            "windows")
+                cargo clean && \
+                cargo build --release
+                readonly ARCHIVE_CMD="7z a -tzip"
+                readonly BIN_SUFFIX=".exe"
+                readonly PKG_NAME="grin-${REPO_TAG}-${JOB_ID}-win-x64"
+                readonly PKG_SUFFIX=".zip"
+                ;;
+
+            *)
+                printf "Error! Unknown \$OS_NAME: \`%s\`" "${OS_NAME}"
+                exit 1
+        esac
+
+        printf "creating package \`%s\` for the release binary...\n" "${PKG_NAME}${PKG_SUFFIX}"
+
+        cd ./target/release/ || exit 1
+        rm -f -- *"${PKG_SUFFIX}"
+        ${ARCHIVE_CMD} "${PKG_NAME}${PKG_SUFFIX}" "grin${BIN_SUFFIX}"
+        ls -ls -- *.tgz | cut -d' ' -f6-
+        openssl md5 "${PKG_NAME}${PKG_SUFFIX}" > "${PKG_NAME}${PKG_SUFFIX}-md5sum.txt"
+        ls -ls -- *-md5sum.txt | cut -d' ' -f6-
+        cd - > /dev/null || exit 1
+
+        printf "%s package \`%s\` generated\n" "${OS_NAME}" "${PKG_NAME}${PKG_SUFFIX}"
+
+        # Generate changelog only on the Linux platform to avoid duplication.
+        [[ "${OS_NAME}" != "linux" ]] && exit 0
+
+        # Generate CHANGELOG.md
+        readonly REPO_SLUG="mimblewimble/grin"
+        readonly REPO_BRANCH="$(git symbolic-ref -q --short HEAD)"
+        readonly REPO_PREV_RELEASE_TAG="$(git describe --abbrev=0 --tags "$(git rev-list --tags --skip=0 --max-count=1)")"
+
+        gem install github_changelog_generator
+
+        # Needed by github_changelog_generator.
+        export CHANGELOG_GITHUB_TOKEN="${VCP_AUTH_TOKEN}"
+
+        github_changelog_generator \
+          --user "$(cut -d "/" -f1 <<< ${REPO_SLUG})" \
+          --project "$(cut -d "/" -f2 <<< ${REPO_SLUG})" \
+          --since-tag "${REPO_PREV_RELEASE_TAG}"
+
+        readonly CHANGELOG_CONTENT="$(<CHANGELOG.md)"
+
+        # Overwrite CHANGELOG.md with JSON data for release patch.
+        jq --null-input \
+          --arg body "${CHANGELOG_CONTENT}" \
+          --arg name "${REPO_TAG}" \
+          --arg tag_name "${REPO_TAG}" \
+          --arg target_commitish "${REPO_BRANCH}" \
+          '{
+            body: $body,
+            name: $name,
+            tag_name: $tag_name,
+            target_commitish: $target_commitish,
+            draft: false,
+            prerelease: false
+          }' > CHANGELOG.md
+
+        readonly HEADERS="Authorization: token ${VCP_AUTH_TOKEN}"
+        readonly RELEASE_URL="https://api.github.com/repos/${REPO_SLUG}/releases"
+        readonly RELEASE_ID="$(curl -0 --request GET -H "${HEADERS}" "${RELEASE_URL}/tags/${REPO_TAG}" 2> /dev/null | grep id | head -n 1 | sed 's/ *"id": *\(.*\),/\1/')"
+
+        printf "updating release changelog %s for repo: %s, branch: %s, release id: %s\n" "${REPO_TAG}" "${REPO_SLUG}" "${REPO_BRANCH}" "${RELEASE_ID}"
+        curl -H "${HEADERS}" --request PATCH --data @CHANGELOG.md "${RELEASE_URL}/${RELEASE_ID}"
+        printf "changelog uploaded.\n"
+        ;;
+esac

.travis.yml

@@ -1,24 +1,28 @@
-language: rust
+# Copyright 2019 The Grin Developers
-
+#
-git:
+# Licensed under the Apache License, Version 2.0 (the "License");
-  depth: false
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 dist: trusty
 sudo: required
 
-cache:
+language: rust
-  cargo: true
-  timeout: 240
-  directories:
-    - $HOME/.cargo
-    - $TRAVIS_BUILD_DIR/target
-
-before_cache:
-  - rm -rf $TRAVIS_BUILD_DIR/target/tmp
 
 rust:
   - stable
 
+git:
+  depth: false
+
 addons:
   apt:
     sources:
@@ -33,6 +37,13 @@ addons:
       - gcc
       - binutils-dev
 
+cache:
+  cargo: true
+  timeout: 240
+  directories:
+    - $HOME/.cargo
+    - $TRAVIS_BUILD_DIR/target
+
 env:
   global:
     - RUST_BACKTRACE="1"
@@ -41,41 +52,29 @@ env:
 matrix:
   include:
     - os: linux
-      env: TEST_SUITE=servers
+      env: CI_JOB="test"        CI_JOB_ARGS="servers"
     - os: linux
-      env: TEST_SUITE=chain-core
+      env: CI_JOB="test"        CI_JOB_ARGS="chain core"
     - os: linux
-      env: TEST_SUITE=pool-p2p-src
+      env: CI_JOB="test"        CI_JOB_ARGS="pool p2p src"
     - os: linux
-      env: TEST_SUITE=keychain-wallet
+      env: CI_JOB="test"        CI_JOB_ARGS="keychain wallet"
     - os: linux
-      env: TEST_SUITE=api-util-store
+      env: CI_JOB="test"        CI_JOB_ARGS="api util store"
+    - os: linux
+      env: CI_JOB="release"     CI_JOB_ARGS=
     - os: osx
-      env: TEST_SUITE=release
+      env: CI_JOB="release"     CI_JOB_ARGS=
+#    - os: windows
+#      env: CI_JOB="release"     CI_JOB_ARGS=
 
-script:
+script: .ci/general-jobs
-  - IFS='-' read -r -a DIRS <<< "$TEST_SUITE"; DIR=${DIRS[0]};
+
-    echo "start testing on folder $DIR...";
+before_cache:
-    if [[ -n "$DIR" ]] && [[ "$TRAVIS_OS_NAME" == "linux" ]];  then cd $DIR && cargo test --release && cd - > /dev/null; fi;
+  - rm -rf $TRAVIS_BUILD_DIR/target/tmp
-  - IFS='-' read -r -a DIRS <<< "$TEST_SUITE"; DIR=${DIRS[1]};
-    if [[ -n "$DIR" ]];  then
-      echo "start testing on folder $DIR...";
-      cd $DIR && cargo test --release && cd - > /dev/null;
-    fi;
-  - IFS='-' read -r -a DIRS <<< "$TEST_SUITE"; DIR=${DIRS[2]};
-    if [[ -n "$DIR" ]];  then
-      echo "start testing on folder $DIR...";
-      cd $DIR && cargo test --release && cd - > /dev/null;
-    fi;
 
 before_deploy:
-  - if [[ "$TEST_SUITE" == "pool-p2p-src" ]]; then
+  - bash .ci/release-jobs
-      cargo clean && cargo build --release && ./.auto-release.sh;
-    fi
-  - if [[ "$TEST_SUITE" == "release" ]]  && [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-      brew update;
-      cargo clean && cargo build --release && ./.auto-release.sh;
-    fi
 
 deploy:
   provider: releases

doc/release_instruction.md

@@ -123,9 +123,9 @@ Remember to replace `0.3.1-pre1` as the real version, and warmly remind the [[Ve
 If you're NOT the owner of the github repo, but at least you have to be a committer which has the right to do a release, the following steps are needed to trigger a version release:
 
 1. Go to release page of the repo, click **Draft a new release**, remember to check the branch is what you're working on! set the **Tag version** to the release number (for    example: `0.3.1-pre1`), and set anything in **Release title** and **description**, then click **Publish release**. Don't worry the title and description parts because we need delete it in next step.
-1. Because github **release** will be auto-created by our `auto-release` building script, we MUST delete the **release** which we just created in previous step! (Unfortunately, there's no way to only create **tag** by web.)
+1. Because github **release** will be auto-created by our `release-jobs` building script, we MUST delete the **release** which we just created in previous step! (Unfortunately, there's no way to only create **tag** by web.)
 
-Even normally Travis-CI need tens of minutes to complete building, I suggest you complete step 2 quickly, otherwise the `auto-release` script will fail on error "release already exist".
+Even normally Travis-CI need tens of minutes to complete building, I suggest you complete step 2 quickly, otherwise the `release-jobs` script will fail on error "release already exist".
 
 ### 2. Travis-CI Building
 
@@ -135,7 +135,7 @@ The release building is just one of the **TEST_DIRS**, named as `none`. So each
 
 So, the point is: the release building job is that one tagged with `TEST_DIR=none`.
 
-Note: `auto-release` script will only be executed on `deploy` stage, and according to Travis-CI, it will be skipped for any **pull-request** trigger, and since we set `tag: true` it will be only executed when triggered by a tag.
+Note: `release-jobs` script will only be executed on `deploy` stage, and according to Travis-CI, it will be skipped for any **pull-request** trigger, and since we set `tag: true` it will be only executed when triggered by a tag.
 
 ### 3. Check the Release Page