ci: run benchmark tests nightly
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
new file mode 100644
index 0000000..7ae14ca
--- /dev/null
+++ b/.github/workflows/benchmark.yml
@@ -0,0 +1,78 @@
+# Benchmark libgit2 against the git reference implementation.
+name: Benchmark
+
+on:
+ workflow_dispatch:
+ schedule:
+ - cron: '15 4 * * *'
+
+jobs:
+ # Run our nightly builds. We build a matrix with the various build
+ # targets and their details. Then we build either in a docker container
+ # (Linux) or on the actual hosts (macOS, Windows).
+ build:
+ strategy:
+ matrix:
+ platform:
+ - name: "Linux (clang, OpenSSL)"
+ env:
+ CC: clang
+ CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DUSE_GSSAPI=ON -DBUILD_TESTS=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI=ON -DCMAKE_BUILD_TYPE=Release
+ CMAKE_BUILD_OPTIONS: --config Release
+ id: linux
+ os: ubuntu-latest
+ setup-script: ubuntu
+ - name: "macOS"
+ os: macos-10.15
+ env:
+ CC: clang
+ CMAKE_OPTIONS: -DREGEX_BACKEND=regcomp_l -DDEPRECATE_HARD=ON -DUSE_GSSAPI=ON -DBUILD_TESTS=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI=ON -DCMAKE_BUILD_TYPE=Release
+ CMAKE_BUILD_OPTIONS: --config Release
+ PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig
+ id: macos
+ setup-script: osx
+ - name: "Windows (amd64, Visual Studio)"
+ os: windows-2019
+ env:
+ ARCH: amd64
+ CMAKE_GENERATOR: Visual Studio 16 2019
+ CMAKE_OPTIONS: -A x64 -DDEPRECATE_HARD=ON -DBUILD_TESTS=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI=ON -DCMAKE_BUILD_TYPE=Release
+ CMAKE_BUILD_OPTIONS: --config Release
+ id: windows
+ setup-script: win32
+ fail-fast: false
+ name: "Build ${{ matrix.platform.name }}"
+ env: ${{ matrix.platform.env }}
+ runs-on: ${{ matrix.platform.os }}
+ steps:
+ - name: Check out repository
+ uses: actions/checkout@v2
+ with:
+ path: source
+ fetch-depth: 0
+ - name: Set up benchmark environment
+ run: source/ci/setup-${{ matrix.platform.setup-script }}-benchmark.sh
+ shell: bash
+ if: matrix.platform.setup-script != ''
+ - name: Build
+ run: |
+ mkdir build && cd build
+ ../source/ci/build.sh
+ shell: bash
+ - name: Benchmark
+ run: |
+ if [[ "$(uname -s)" == MINGW* ]]; then
+ GIT2_CLI="$(cygpath -w $(pwd))\\build\\Release\\git2_cli"
+ else
+ GIT2_CLI="$(pwd)/build/git2_cli"
+ fi
+
+ mkdir benchmark && cd benchmark
+ ../source/tests/benchmarks/benchmark.sh --baseline-cli "git" --cli "${GIT2_CLI}" --json benchmarks.json --zip benchmarks.zip
+ shell: bash
+ - name: Upload results
+ uses: actions/upload-artifact@v2
+ with:
+ name: benchmark-${{ matrix.platform.id }}
+ path: benchmark
+ if: always()
diff --git a/ci/setup-osx-benchmark.sh b/ci/setup-osx-benchmark.sh
new file mode 100755
index 0000000..80d8768
--- /dev/null
+++ b/ci/setup-osx-benchmark.sh
@@ -0,0 +1,6 @@
+#!/bin/sh
+
+set -ex
+
+brew update
+brew install hyperfine
diff --git a/ci/setup-ubuntu-benchmark.sh b/ci/setup-ubuntu-benchmark.sh
new file mode 100755
index 0000000..561a18f
--- /dev/null
+++ b/ci/setup-ubuntu-benchmark.sh
@@ -0,0 +1,20 @@
+#!/bin/sh
+
+set -ex
+
+sudo apt-get update
+sudo apt-get install -y --no-install-recommends \
+ cargo \
+ cmake \
+ gcc \
+ git \
+ krb5-user \
+ libkrb5-dev \
+ libssl-dev \
+ libz-dev \
+ make \
+ ninja-build \
+ pkgconf
+
+wget https://github.com/sharkdp/hyperfine/releases/download/v1.12.0/hyperfine_1.12.0_amd64.deb
+sudo dpkg -i hyperfine_1.12.0_amd64.deb
diff --git a/ci/setup-win32-benchmark.sh b/ci/setup-win32-benchmark.sh
new file mode 100755
index 0000000..0eac2f6
--- /dev/null
+++ b/ci/setup-win32-benchmark.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+set -ex
+
+choco install hyperfine zip
+
+CHOCO_PATH=$(mktemp -d)
+curl -L https://github.com/ethomson/PurgeStandbyList/releases/download/v1.0/purgestandbylist.1.0.0.nupkg -o "${CHOCO_PATH}/purgestandbylist.1.0.0.nupkg"
+choco install purgestandbylist -s $(cygpath -w "${CHOCO_PATH}")