Commit 51e6eada authored by Dominik Charousset's avatar Dominik Charousset

Add coverage report to build pipeline

parent 13095ce9
#!/usr/bin/env groovy
// Default CMake flags for most builds (except coverage).
defaultBuildFlags = [
'CAF_MORE_WARNINGS:BOOL=yes',
// Default CMake flags for release builds.
defaultReleaseBuildFlags = [
'CAF_ENABLE_RUNTIME_CHECKS:BOOL=yes',
'CAF_NO_OPENCL:BOOL=yes',
'CAF_INSTALL_UNIT_TESTS:BOOL=yes',
]
// CMake flags for release builds.
releaseBuildFlags = defaultBuildFlags + [
]
// CMake flags for debug builds.
debugBuildFlags = defaultBuildFlags + [
// Default CMake flags for debug builds.
defaultDebugBuildFlags = defaultReleaseBuildFlags + [
'CAF_ENABLE_RUNTIME_CHECKS:BOOL=yes',
'CAF_ENABLE_ADDRESS_SANITIZER:BOOL=yes',
'CAF_LOG_LEVEL:STRING=TRACE',
]
// CMake flags for macOS builds only.
macBuildFlags = [
defaultBuildFlags = [
debug: defaultDebugBuildFlags,
release: defaultReleaseBuildFlags,
]
// CMake flags by OS and build type.
buildFlags = [
macOS: [
debug: defaultDebugBuildFlags + [
'OPENSSL_ROOT_DIR=/usr/local/opt/openssl',
'OPENSSL_INCLUDE_DIR=/usr/local/opt/openssl/include',
],
release: defaultReleaseBuildFlags + [
'OPENSSL_ROOT_DIR=/usr/local/opt/openssl',
'OPENSSL_INCLUDE_DIR=/usr/local/opt/openssl/include',
],
],
Windows: [
debug: defaultDebugBuildFlags + [
'CAF_BUILD_STATIC_ONLY:BOOL=yes',
],
release: defaultReleaseBuildFlags + [
'CAF_BUILD_STATIC_ONLY:BOOL=yes',
],
],
]
// Our build matrix. The keys are the operating system labels and the values
// are lists of tool labels.
buildMatrix = [
// Debug builds with ASAN + logging for various OS/compiler combinations.
['Linux', [
builds: ['debug'],
tools: ['gcc4.8', 'gcc4.9', 'gcc5', 'gcc6', 'gcc7', 'gcc8'],
cmakeArgs: debugBuildFlags,
tools: ['gcc4.8', 'gcc4.9', 'gcc5', 'gcc6', 'gcc7'],
]],
['macOS', [
['Linux', [
builds: ['debug'],
tools: ['clang'],
cmakeArgs: debugBuildFlags + macBuildFlags,
tools: ['gcc8'],
extraSteps: ['coverageReport'],
]],
// One release build per supported OS. FreeBSD and Windows have the least
// testing outside Jenkins, so we also explicitly schedule debug builds.
['Linux', [
builds: ['release'],
tools: ['gcc8', 'clang'],
cmakeArgs: releaseBuildFlags,
]],
['macOS', [
builds: ['release'],
builds: ['debug', 'release'],
tools: ['clang'],
cmakeArgs: releaseBuildFlags + macBuildFlags,
]],
['FreeBSD', [
builds: ['debug'], // no release build for now, because it takes 1h
tools: ['clang'],
cmakeArgs: debugBuildFlags,
]],
['Windows', [
builds: ['debug', 'release'],
// TODO: debug build currently broken
//builds: ['debug', 'release'],
builds: ['release'],
tools: ['msvc'],
cmakeArgs: [
'CAF_BUILD_STATIC_ONLY:BOOL=yes',
'CAF_ENABLE_RUNTIME_CHECKS:BOOL=yes',
'CAF_NO_OPENCL:BOOL=yes',
],
]],
// One Additional build for coverage reports.
/* TODO: this build exhausts all storage on the node and is temporarily
* disabled until resolving the issue
['Linux', [
builds: ['debug'],
tools: ['gcc8 && gcovr'],
extraSteps: ['coverageReport'],
cmakeArgs: [
'CAF_ENABLE_GCOV:BOOL=yes',
'CAF_NO_EXCEPTIONS:BOOL=yes',
'CAF_FORCE_NO_EXCEPTIONS:BOOL=yes',
],
]],
*/
]
// Optional environment variables for combinations of labels.
......@@ -86,15 +78,69 @@ buildEnvironments = [
'Linux && clang': ['CXX=clang++'],
]
// Called *after* a build succeeded.
def coverageReport() {
// Adds additional context information to commits on GitHub.
def setBuildStatus(context, state, message) {
echo "set ${context} result for commit ${env.GIT_COMMIT} to $state: $message"
step([
$class: 'GitHubCommitStatusSetter',
commitShaSource: [
$class: 'ManuallyEnteredShaSource',
sha: env.GIT_COMMIT,
],
reposSource: [
$class: 'ManuallyEnteredRepositorySource',
url: env.GIT_URL,
],
contextSource: [
$class: 'ManuallyEnteredCommitContextSource',
context: context,
],
errorHandlers: [[
$class: 'ChangingBuildStatusErrorHandler',
result: 'SUCCESS',
]],
statusResultSource: [
$class: 'ConditionalStatusResultSource',
results: [[
$class: 'AnyBuildResult',
state: state,
message: message,
]]
],
]);
}
// Creates coverage reports via the Cobertura plugin.
def coverageReport(buildId) {
echo "Create coverage report for build ID $buildId"
// Paths we wish to ignore in the coverage report.
def installDir = "$WORKSPACE/$buildId"
def excludePaths = [
"/usr/",
"$WORKSPACE/caf-sources/examples",
"$WORKSPACE/caf-sources/tools",
"$WORKSPACE/caf-sources/libcaf_test",
"$WORKSPACE/caf-sources/libcaf_core/test",
"$WORKSPACE/caf-sources/libcaf_io/test",
"$WORKSPACE/caf-sources/libcaf_openssl/test",
"$WORKSPACE/caf-sources/libcaf_opencl",
"$WORKSPACE/caf-sources/libcaf_core/caf/scheduler/profiled_coordinator.hpp",
]
def excludePathsStr = excludePaths.join(',')
dir('caf-sources') {
sh 'gcovr -e examples -e tools -e libcaf_test -e ".*/test/.*" -e libcaf_core/caf/scheduler/profiled_coordinator.hpp -x -r . > coverage.xml'
archiveArtifacts '**/coverage.xml'
try {
withEnv(['ASAN_OPTIONS=verify_asan_link_order=false:detect_leaks=0']) {
sh """
kcov --exclude-path=$excludePathsStr kcov-result ./build/caf-test &> kcov_output.txt
find . -name 'coverage.json' -exec mv {} result.json \\;
"""
}
stash includes: 'result.json', name: 'coverage-result'
archiveArtifacts '**/cobertura.xml'
cobertura([
autoUpdateHealth: false,
autoUpdateStability: false,
coberturaReportFile: '**/coverage.xml',
coberturaReportFile: '**/cobertura.xml',
conditionalCoverageTargets: '70, 0, 0',
failUnhealthy: false,
failUnstable: false,
......@@ -105,9 +151,14 @@ def coverageReport() {
sourceEncoding: 'ASCII',
zoomCoverageChart: false,
])
} catch (Exception e) {
echo "exception: $e"
archiveArtifacts 'kcov_output.txt'
}
}
}
// Wraps `cmakeBuild`.
def cmakeSteps(buildType, cmakeArgs, buildId) {
def installDir = "$WORKSPACE/$buildId"
dir('caf-sources') {
......@@ -126,11 +177,17 @@ def cmakeSteps(buildType, cmakeArgs, buildId) {
]],
])
// Run unit tests.
try {
ctest([
arguments: '--output-on-failure',
installation: 'cmake in search path',
workingDir: 'build',
])
writeFile file: "${buildId}.success", text: "success\n"
} catch (Exception) {
writeFile file: "${buildId}.failure", text: "failure\n"
}
stash includes: "${buildId}.*", name: buildId
}
// Only generate artifacts for the master branch.
if (PrettyJobBaseName == 'master') {
......@@ -157,15 +214,14 @@ def buildSteps(buildType, cmakeArgs, buildId) {
}
} else {
echo "Unix build on $NODE_NAME"
withEnv(["label_exp=" + STAGE_NAME.toLowerCase(),
"ASAN_OPTIONS=detect_leaks=0"]) {
withEnv(["label_exp=" + STAGE_NAME.toLowerCase(), "ASAN_OPTIONS=detect_leaks=0"]) {
cmakeSteps(buildType, cmakeArgs, buildId)
}
}
}
// Builds a stage for given builds. Results in a parallel stage `if builds.size() > 1`.
def makeBuildStages(matrixIndex, builds, lblExpr, settings) {
def makeBuildStages(matrixIndex, os, builds, lblExpr, settings) {
builds.collectEntries { buildType ->
def id = "$matrixIndex $lblExpr: $buildType"
[
......@@ -176,8 +232,8 @@ def makeBuildStages(matrixIndex, builds, lblExpr, settings) {
try {
def buildId = "${lblExpr}_${buildType}".replace(' && ', '_')
withEnv(buildEnvironments[lblExpr] ?: []) {
buildSteps(buildType, settings['cmakeArgs'], buildId)
(settings['extraSteps'] ?: []).each { fun -> "$fun"() }
buildSteps(buildType, (buildFlags[os] ?: defaultBuildFlags)[buildType], buildId)
(settings['extraSteps'] ?: []).each { fun -> "$fun"(buildId) }
}
} finally {
cleanWs()
......@@ -191,7 +247,7 @@ def makeBuildStages(matrixIndex, builds, lblExpr, settings) {
pipeline {
options {
buildDiscarder(logRotator(numToKeepStr: '50', artifactNumToKeepStr: '10'))
buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '5'))
}
agent none
environment {
......@@ -208,7 +264,16 @@ pipeline {
dir('caf-sources') {
checkout scm
sh './scripts/get-release-version.sh'
/// We really don't need to copy the .git folder around.
dir('.git') {
deleteDir()
}
}
zip([
archive: true,
dir: 'caf-sources',
zipFile: 'sources.zip',
])
stash includes: 'caf-sources/**', name: 'caf-sources'
}
}
......@@ -224,13 +289,58 @@ pipeline {
def matrixIndex = "[$index:$toolIndex]"
def builds = settings['builds']
def labelExpr = "$os && $tool"
xs << makeBuildStages(matrixIndex, builds, labelExpr, settings)
xs << makeBuildStages(matrixIndex, os, builds, labelExpr, settings)
}
}
parallel xs
}
}
}
stage('Check Test Results') {
agent { label 'master' }
steps {
script {
dir('tmp') {
// Compute the list of all build IDs.
def buildIds = []
buildMatrix.each { entry ->
def (os, settings) = entry
settings['tools'].each { tool ->
settings['builds'].each {
buildIds << "${os}_${tool}_${it}"
}
}
}
// Compute how many tests have succeeded
def builds = buildIds.size()
def successes = buildIds.inject(0) { result, buildId ->
try { unstash buildId }
catch (Exception) { }
result + (fileExists("${buildId}.success") ? 1 : 0)
}
echo "$successes unit tests tests of $builds were successful"
if (builds == successes) {
setBuildStatus('unit-tests', 'SUCCESS', 'All builds passed the unit tests')
} else {
def failures = builds - successes
setBuildStatus('unit-tests', 'FAILURE', "$failures/$builds builds failed to run the unit tests")
}
// Get the coverage result.
try {
unstash 'coverage-result'
if (fileExists('result.json')) {
def resultJson = readJSON file: 'result.json'
setBuildStatus('coverage', 'SUCCESS', resultJson['percent_covered'] + '% coverage')
} else {
setBuildStatus('coverage', 'FAILURE', 'Unable to get coverage report')
}
} catch (Exception) {
setBuildStatus('coverage', 'FAILURE', 'Unable to generate coverage report')
}
}
}
}
}
stage('Documentation') {
agent { label 'pandoc' }
steps {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment