From b8c97997651d46b379f6b144788d12bd517434e7 Mon Sep 17 00:00:00 2001 From: Kjeld Schouten-Lebbing Date: Sat, 4 Dec 2021 15:20:01 +0100 Subject: [PATCH] feat(ci): Run release pipeline in parallel (#1463) * move actual running to function * add some test charts * prepare for parallel execution * enable parallel execution * use newer devcontainer * use bash subshell for helm dependency update * export all the things * use j2 for parallel * whoops * helm template devnull * use correct tooling * questions and catalog functions export * Revert "add some test charts" This reverts commit d9acc26d6a47886d95ed76929a66fd547e16fcef. * use correct variables * nproc*2 * dev null the verbose helm template output * some trivy scanning cleanup * fix small mistake --- .github/workflows/apps.test.yaml | 4 +- tools/build-release.sh | 99 +++++++++++++++++++++----------- 2 files changed, 66 insertions(+), 37 deletions(-) diff --git a/.github/workflows/apps.test.yaml b/.github/workflows/apps.test.yaml index 6ef94ee61a8..4ca0bef3a56 100644 --- a/.github/workflows/apps.test.yaml +++ b/.github/workflows/apps.test.yaml @@ -159,7 +159,7 @@ jobs: release-test: runs-on: ubuntu-latest container: - image: ghcr.io/truecharts/devcontainer:v2.1.0 + image: ghcr.io/truecharts/devcontainer:v2.1.1 steps: - name: Cache helm repo cache id: cache @@ -183,4 +183,4 @@ jobs: - name: build-and-run run: | - find ./ -type f -name *.yaml -exec sed -i 's/tccr.io/ghcr.io/gI' {} \; && tools/build-release.sh --config .github/cr.yaml + tools/build-release.sh --config .github/cr.yaml diff --git a/tools/build-release.sh b/tools/build-release.sh index 7c1d2c64750..21f16e1f523 100755 --- a/tools/build-release.sh +++ b/tools/build-release.sh @@ -30,6 +30,7 @@ main() { local production= local charts_repo_url= local token=${CR_TOKEN:-false} + local parthreads=$(($(nproc) * 2)) parse_command_line "$@" if [ "${token}" == "false" ]; then @@ -58,37 +59,8 @@ main() { mkdir -p .cr-index prep_helm - for chart in "${changed_charts[@]}"; do - if [[ -d "$chart" ]]; then - echo "Start processing $chart ..." - chartversion=$(cat ${chart}/Chart.yaml | grep "^version: " | awk -F" " '{ print $2 }') - chartname=$(basename ${chart}) - train=$(basename $(dirname "$chart")) - SCALESUPPORT=$(cat ${chart}/Chart.yaml | yq '.annotations."truecharts.org/SCALE-support"' -r) - sync_tag "$chart" "$chartname" "$train" "$chartversion" || echo "Tag sync failed..." - helm dependency update "${chart}" --skip-refresh || sleep 10 && helm dependency update "${chart}" --skip-refresh || sleep 10 && helm dependency update "${chart}" --skip-refresh - helm_sec_scan "$chart" "$chartname" "$train" "$chartversion" || echo "helm-chart security-scan failed..." - container_sec_scan "$chart" "$chartname" "$train" "$chartversion" || echo "container security-scan failed..." - sec_scan_cleanup "$chart" "$chartname" "$train" "$chartversion" || echo "security-scan cleanup failed..." - create_changelog "$chart" "$chartname" "$train" "$chartversion" || echo "changelog generation failed..." - generate_docs "$chart" "$chartname" "$train" "$chartversion" || echo "Docs generation failed..." - copy_docs "$chart" "$chartname" "$train" "$chartversion" || echo "Docs Copy failed..." - helm dependency update "${chart}" --skip-refresh || sleep 10 && helm dependency update "${chart}" --skip-refresh || sleep 10 && helm dependency update "${chart}" --skip-refresh - package_chart "$chart" - if [[ "${SCALESUPPORT}" == "true" ]]; then - clean_apps "$chart" "$chartname" "$train" "$chartversion" - copy_apps "$chart" "$chartname" "$train" "$chartversion" - patch_apps "$chart" "$chartname" "$train" "$chartversion" - include_questions "$chart" "$chartname" "$train" "$chartversion" - clean_catalog "$chart" "$chartname" "$train" "$chartversion" - else - echo "Skipping chart ${chart}, no correct SCALE compatibility layer detected" - fi - else - echo "Chart '$chart' no longer exists in repo. Skipping it..." - fi - echo "Done processing $chart ..." - done + + parallel -j ${parthreads} chart_runner ::: ${changed_charts[@]} echo "Starting post-processing" pre_commit validate_catalog @@ -111,6 +83,38 @@ main() { popd > /dev/null } +chart_runner(){ + if [[ -d "${1}" ]]; then + echo "Start processing ${1} ..." + chartversion=$(cat ${1}/Chart.yaml | grep "^version: " | awk -F" " '{ print $2 }') + chartname=$(basename ${1}) + train=$(basename $(dirname "${1}")) + SCALESUPPORT=$(cat ${1}/Chart.yaml | yq '.annotations."truecharts.org/SCALE-support"' -r) + helm dependency update "${1}" --skip-refresh || (sleep 10 && helm dependency update "${1}" --skip-refresh) || (sleep 10 && helm dependency update "${1}" --skip-refresh) + helm_sec_scan "${1}" "${chartname}" "$train" "${chartversion}" || echo "helm-chart security-scan failed..." + container_sec_scan "${1}" "${chartname}" "$train" "${chartversion}" || echo "container security-scan failed..." + sec_scan_cleanup "${1}" "${chartname}" "$train" "${chartversion}" || echo "security-scan cleanup failed..." + sync_tag "${1}" "${chartname}" "$train" "${chartversion}" || echo "Tag sync failed..." + create_changelog "${1}" "${chartname}" "$train" "${chartversion}" || echo "changelog generation failed..." + generate_docs "${1}" "${chartname}" "$train" "${chartversion}" || echo "Docs generation failed..." + copy_docs "${1}" "${chartname}" "$train" "${chartversion}" || echo "Docs Copy failed..." + package_chart "${1}" + if [[ "${SCALESUPPORT}" == "true" ]]; then + clean_apps "${1}" "${chartname}" "$train" "${chartversion}" + copy_apps "${1}" "${chartname}" "$train" "${chartversion}" + patch_apps "${1}" "${chartname}" "$train" "${chartversion}" + include_questions "${1}" "${chartname}" "$train" "${chartversion}" + clean_catalog "${1}" "${chartname}" "$train" "${chartversion}" + else + echo "Skipping chart ${1}, no correct SCALE compatibility layer detected" + fi + else + echo "Chart '${1}' no longer exists in repo. Skipping it..." + fi + echo "Done processing ${1} ..." +} +export -f chart_runner + include_questions(){ local chart="$1" local chartname="$2" @@ -235,6 +239,7 @@ include_questions(){ 1' templates/questions/resources.yaml ${target}/questions.yaml > tmp && mv tmp ${target}/questions.yaml } +export -f include_questions clean_catalog() { local chart="$1" @@ -251,6 +256,7 @@ clean_catalog() { done cd - } +export -f clean_catalog # Designed to ensure the appversion in Chart.yaml is in sync with the primary App tag if found sync_tag() { @@ -276,6 +282,7 @@ sync_tag() { tag="${tag%.}" sed -i -e "s|appVersion: .*|appVersion: \"${tag}\"|" "${chart}/Chart.yaml" } +export -f sync_tag helm_sec_scan() { local chart="$1" @@ -291,10 +298,11 @@ helm_sec_scan() { echo "" >> ${chart}/sec-scan.md echo "##### Scan Results" >> ${chart}/sec-scan.md echo "" >> ${chart}/sec-scan.md - helm template ${chart} --output-dir ${chart}/render + helm template ${chart} --output-dir ${chart}/render > /dev/null trivy config -f template --template "@./templates/trivy.tpl" ${chart}/render >> ${chart}/sec-scan.md echo "" >> ${chart}/sec-scan.md } + export -f helm_sec_scan container_sec_scan() { local chart="$1" @@ -306,20 +314,22 @@ container_sec_scan() { echo "" >> ${chart}/sec-scan.md echo "##### Detected Containers" >> ${chart}/sec-scan.md echo "" >> ${chart}/sec-scan.md - find ${chart}/render/ -name '*.yaml' -type f -exec cat {} \; | grep image: | sed "s/image: //g" | sed "s/\"//g" >> ${chart}/render/containers.tmp + find ./${chart}/render/ -name '*.yaml' -type f -exec cat {} \; | grep image: | sed "s/image: //g" | sed "s/\"//g" >> ${chart}/render/containers.tmp cat ${chart}/render/containers.tmp >> ${chart}/sec-scan.md echo "" >> ${chart}/sec-scan.md echo "##### Scan Results" >> ${chart}/sec-scan.md echo "" >> ${chart}/sec-scan.md for container in $(cat ${chart}/render/containers.tmp); do + ghcrcont="$(echo ${container} | sed 's/tccr.io/ghcr.io/g')" + echo "processing container: ${container} using ${ghcrcont}" echo "**Container: ${container}**" >> ${chart}/sec-scan.md echo "" >> ${chart}/sec-scan.md - ghcrcont=$(echo ${container} | sed "s/tccr.io/ghcr.io/g") - trivy image -f template --template "@./templates/trivy.tpl" ${ghcrcont} >> ${chart}/sec-scan.md + trivy image -f template --template "@./templates/trivy.tpl" "${ghcrcont}" >> ${chart}/sec-scan.md echo "" >> ${chart}/sec-scan.md done } + export -f container_sec_scan sec_scan_cleanup() { local chart="$1" @@ -329,6 +339,7 @@ sec_scan_cleanup() { rm -rf ${chart}/render sed -i 's/ghcr.io/tccr.io/g' ${chart}/sec-scan.md } + export -f sec_scan_cleanup pre_commit() { if [[ -z "$standalone" ]]; then @@ -338,6 +349,7 @@ pre_commit() { find . -name '*.sh' | xargs chmod +x fi } + export -f pre_commit edit_release() { local chart="$1" @@ -347,6 +359,7 @@ edit_release() { # In here we can in the future add code to edit the release notes of the github releases # For example: using the github API: https://docs.github.com/en/rest/reference/repos#update-a-release } + export -f edit_release create_changelog() { local chart="$1" @@ -371,6 +384,7 @@ create_changelog() { rm ${chart}/app-changelog.md || echo "changelog not found..." fi } + export -f create_changelog copy_general_docs() { yes | cp -rf index.yaml docs/index.yaml 2>/dev/null || : @@ -385,6 +399,7 @@ copy_general_docs() { yes | cp -rf NOTICE docs/about/legal/NOTICE.md 2>/dev/null || : sed -i '1s/^/# NOTICE
\n\n/' docs/about/legal/NOTICE.md } + export -f copy_general_docs generate_docs() { local chart="$1" @@ -423,6 +438,7 @@ generate_docs() { fi fi } + export -f generate_docs copy_docs() { @@ -447,6 +463,7 @@ copy_docs() { sed -i '1s/^/# License
\n\n/' docs/apps/${train}/${chartname}/LICENSE.md 2>/dev/null || : fi } + export -f copy_docs prep_helm() { if [[ -z "$standalone" ]]; then @@ -458,6 +475,7 @@ prep_helm() { helm repo update fi } + export -f prep_helm clean_apps() { local chart="$1" @@ -468,6 +486,7 @@ clean_apps() { rm -Rf catalog/${train}/${chartname}/${chartversion} 2>/dev/null || : rm -Rf catalog/${train}/${chartname}/item.yaml 2>/dev/null || : } +export -f clean_apps patch_apps() { local chart="$1" @@ -495,6 +514,7 @@ patch_apps() { echo "" >> ${target}/app-readme.md echo "This App is supplied by TrueCharts, for more information please visit https://truecharts.org" >> ${target}/app-readme.md } +export -f patch_apps copy_apps() { local chart="$1" @@ -506,6 +526,7 @@ copy_apps() { cp -Rf ${chart}/* catalog/${train}/${chartname}/${chartversion}/ } +export -f copy_apps validate_catalog() { if [[ -z "$standalone" ]]; then @@ -513,6 +534,7 @@ validate_catalog() { /usr/local/bin/catalog_validate validate --path "${PWD}/catalog" fi } +export -f validate_catalog parse_command_line() { @@ -609,6 +631,7 @@ parse_command_line() { charts_repo_url="https://$owner.github.io/$repo" fi } +export -f parse_command_line lookup_latest_tag() { git fetch --tags > /dev/null 2>&1 @@ -617,6 +640,7 @@ lookup_latest_tag() { git rev-list --max-parents=0 --first-parent HEAD fi } +export -f lookup_latest_tag filter_charts() { while read -r chart; do @@ -628,6 +652,7 @@ filter_charts() { fi done } +export -f filter_charts lookup_changed_charts() { local commit="$1" @@ -640,6 +665,7 @@ lookup_changed_charts() { cut -d '/' -f "$fields" <<< "$changed_files" | uniq | filter_charts } +export -f lookup_changed_charts package_chart() { local chart="$1" @@ -653,6 +679,7 @@ package_chart() { cr package "${args[@]}" fi } +export -f package_chart release_charts() { local args=(-o "$owner" -r "$repo" -c "$(git rev-parse HEAD)") @@ -665,6 +692,7 @@ release_charts() { cr upload "${args[@]}" fi } +export -f release_charts update_index() { local args=(-o "$owner" -r "$repo" -c "$charts_repo_url") @@ -677,5 +705,6 @@ update_index() { cr index "${args[@]}" fi } +export -f update_index main "$@"