feat(ci): split docs generation from catalog generation
This commit is contained in:
kjeld Schouten-Lebbing
@@ -0,0 +1,298 @@
|
||||
#!/usr/bin/env bash
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
|
||||
DEFAULT_CHART_RELEASER_VERSION=v1.2.1
|
||||
|
||||
show_help() {
|
||||
cat << EOF
|
||||
Usage: $(basename "$0") <options>
|
||||
-h, --help Display help
|
||||
-d, --charts-dir The charts directory (default: charts)
|
||||
EOF
|
||||
}
|
||||
|
||||
main() {
|
||||
local charts_dir=charts/**
|
||||
local token=${CR_TOKEN:-false}
|
||||
local parthreads=$(($(nproc) * 2))
|
||||
|
||||
parse_command_line "$@"
|
||||
|
||||
local repo_root
|
||||
repo_root=$(git rev-parse --show-toplevel)
|
||||
pushd "$repo_root" > /dev/null
|
||||
|
||||
echo 'Looking up latest tag...'
|
||||
local latest_tag
|
||||
latest_tag=$(lookup_latest_tag)
|
||||
|
||||
echo "Discovering changed charts since '$latest_tag'..."
|
||||
local changed_charts=()
|
||||
readarray -t changed_charts <<< "$(lookup_changed_charts "$latest_tag")"
|
||||
if [[ -n "${changed_charts[*]}" ]]; then
|
||||
prep_helm
|
||||
parallel -j ${parthreads} chart_runner '2>&1' ::: ${changed_charts[@]}
|
||||
echo "Starting post-processing"
|
||||
pre_commit
|
||||
else
|
||||
echo "Nothing to do. No chart changes detected."
|
||||
fi
|
||||
|
||||
popd > /dev/null
|
||||
}
|
||||
|
||||
chart_runner(){
|
||||
if [[ -d "${1}" ]]; then
|
||||
echo "Start processing ${1} ..."
|
||||
chartversion=$(cat ${1}/Chart.yaml | grep "^version: " | awk -F" " '{ print $2 }')
|
||||
chartname=$(basename ${1})
|
||||
train=$(basename $(dirname "${1}"))
|
||||
SCALESUPPORT=$(cat ${1}/Chart.yaml | yq '.annotations."truecharts.org/SCALE-support"' -r)
|
||||
helm dependency update "${1}" --skip-refresh || (sleep 10 && helm dependency update "${1}" --skip-refresh) || (sleep 10 && helm dependency update "${1}" --skip-refresh)
|
||||
helm_sec_scan "${1}" "${chartname}" "$train" "${chartversion}" || echo "helm-chart security-scan failed..."
|
||||
container_sec_scan "${1}" "${chartname}" "$train" "${chartversion}" || echo "container security-scan failed..."
|
||||
sec_scan_cleanup "${1}" "${chartname}" "$train" "${chartversion}" || echo "security-scan cleanup failed..."
|
||||
sync_tag "${1}" "${chartname}" "$train" "${chartversion}" || echo "Tag sync failed..."
|
||||
create_changelog "${1}" "${chartname}" "$train" "${chartversion}" || echo "changelog generation failed..."
|
||||
generate_docs "${1}" "${chartname}" "$train" "${chartversion}" || echo "Docs generation failed..."
|
||||
else
|
||||
echo "Chart '${1}' no longer exists in repo. Skipping it..."
|
||||
fi
|
||||
echo "Done processing ${1} ..."
|
||||
}
|
||||
export -f chart_runner
|
||||
|
||||
prep_helm() {
|
||||
if [[ -z "$standalone" ]]; then
|
||||
helm repo add truecharts-old https://truecharts.org
|
||||
helm repo add truecharts https://charts.truecharts.org
|
||||
helm repo add truecharts-library https://library-charts.truecharts.org
|
||||
helm repo add bitnami https://charts.bitnami.com/bitnami
|
||||
helm repo add metallb https://metallb.github.io/metallb
|
||||
helm repo add grafana https://grafana.github.io/helm-charts
|
||||
helm repo add prometheus https://prometheus-community.github.io/helm-charts
|
||||
helm repo add amd-gpu-helm https://radeonopencompute.github.io/k8s-device-plugin/
|
||||
helm repo update
|
||||
fi
|
||||
}
|
||||
export -f prep_helm
|
||||
|
||||
# Designed to ensure the appversion in Chart.yaml is in sync with the primary App tag if found
|
||||
sync_tag() {
|
||||
local chart="$1"
|
||||
local chartname="$2"
|
||||
local train="$3"
|
||||
local chartversion="$4"
|
||||
echo "Attempting to sync primary tag with appversion for: ${chartname}"
|
||||
local tag="$(cat ${chart}/values.yaml | grep '^ tag: ' | awk -F" " '{ print $2 }' | head -1)"
|
||||
tag="${tag%%@*}"
|
||||
tag="${tag:-auto}"
|
||||
tag=$(echo $tag | sed "s/release-//g")
|
||||
tag=$(echo $tag | sed "s/release_//g")
|
||||
tag=$(echo $tag | sed "s/version-//g")
|
||||
tag=$(echo $tag | sed "s/version_//g")
|
||||
tag="${tag#*V.}"
|
||||
tag="${tag#*v-}"
|
||||
tag="${tag#*v}"
|
||||
tag="${tag%-*}"
|
||||
tag="${tag:0:10}"
|
||||
tag="${tag%-}"
|
||||
tag="${tag%_}"
|
||||
tag="${tag%.}"
|
||||
sed -i -e "s|appVersion: .*|appVersion: \"${tag}\"|" "${chart}/Chart.yaml"
|
||||
}
|
||||
export -f sync_tag
|
||||
|
||||
helm_sec_scan() {
|
||||
local chart="$1"
|
||||
local chartname="$2"
|
||||
local train="$3"
|
||||
local chartversion="$4"
|
||||
echo "Scanning helm security for ${chartname}"
|
||||
mkdir -p ${chart}/render
|
||||
rm -rf ${chart}/security.md || echo "removing old security.md file failed..."
|
||||
cat templates/security.tpl >> ${chart}/security.md
|
||||
echo "" >> ${chart}/security.md
|
||||
helm template ${chart} --output-dir ${chart}/render > /dev/null
|
||||
trivy config -f template --template "@./templates/trivy-config.tpl" -o ${chart}/render/tmpsec${chartname}.md ${chart}/render
|
||||
cat ${chart}/render/tmpsec${chartname}.md >> ${chart}/security.md
|
||||
rm -rf ${chart}/render/tmpsec${chartname}.md || true
|
||||
echo "" >> ${chart}/security.md
|
||||
}
|
||||
export -f helm_sec_scan
|
||||
|
||||
container_sec_scan() {
|
||||
local chart="$1"
|
||||
local chartname="$2"
|
||||
local train="$3"
|
||||
local chartversion="$4"
|
||||
echo "Scanning container security for ${chartname}"
|
||||
echo "## Containers" >> ${chart}/security.md
|
||||
echo "" >> ${chart}/security.md
|
||||
echo "##### Detected Containers" >> ${chart}/security.md
|
||||
echo "" >> ${chart}/security.md
|
||||
find ./${chart}/render/ -name '*.yaml' -type f -exec cat {} \; | grep image: | sed "s/image: //g" | sed "s/\"//g" >> ${chart}/render/containers.tmp
|
||||
cat ${chart}/render/containers.tmp >> ${chart}/security.md
|
||||
echo "" >> ${chart}/security.md
|
||||
echo "##### Scan Results" >> ${chart}/security.md
|
||||
echo "" >> ${chart}/security.md
|
||||
for container in $(cat ${chart}/render/containers.tmp); do
|
||||
echo "processing container: ${container}"
|
||||
echo "" >> ${chart}/security.md
|
||||
trivy image -f template --template "@./templates/trivy-container.tpl" -o ${chart}/render/tmpsec${chartname}.md "${container}"
|
||||
cat ${chart}/render/tmpsec${chartname}.md >> ${chart}/security.md
|
||||
rm -rf ${chart}/render/tmpsec${chartname}.md || true
|
||||
echo "" >> ${chart}/security.md
|
||||
done
|
||||
|
||||
}
|
||||
export -f container_sec_scan
|
||||
|
||||
sec_scan_cleanup() {
|
||||
local chart="$1"
|
||||
local chartname="$2"
|
||||
local train="$3"
|
||||
local chartversion="$4"
|
||||
rm -rf ${chart}/render
|
||||
sed -i 's/ghcr.io/tccr.io/g' ${chart}/security.md
|
||||
}
|
||||
export -f sec_scan_cleanup
|
||||
|
||||
pre_commit() {
|
||||
if [[ -z "$standalone" ]]; then
|
||||
echo "Running pre-commit test-and-cleanup..."
|
||||
pre-commit run --all ||:
|
||||
# Fix sh files to always be executable
|
||||
find . -name '*.sh' | xargs chmod +x
|
||||
fi
|
||||
}
|
||||
export -f pre_commit
|
||||
|
||||
create_changelog() {
|
||||
local chart="$1"
|
||||
local chartname="$2"
|
||||
local train="$3"
|
||||
local chartversion="$4"
|
||||
local prevversion="$(git tag -l "${chartname}-*" --sort=-v:refname | head -n 1)"
|
||||
if [[ -z "$standalone" ]]; then
|
||||
echo "Generating changelogs for: ${chartname}"
|
||||
# SCALE "Changelog" containing only last change
|
||||
git-chglog --next-tag ${chartname}-${chartversion} --tag-filter-pattern ${chartname} --path ${chart} -o ${chart}/app-changelog.md ${chartname}-${chartversion}
|
||||
# Append SCALE changelog to actual changelog
|
||||
|
||||
if [[ -f "${chart}/CHANGELOG.md" ]]; then
|
||||
true
|
||||
else
|
||||
touch ${chart}/CHANGELOG.md
|
||||
fi
|
||||
sed -i '1d' ${chart}/CHANGELOG.md
|
||||
cat ${chart}/app-changelog.md | cat - ${chart}/CHANGELOG.md > temp && mv temp ${chart}/CHANGELOG.md
|
||||
sed -i '1s/^/# Changelog<br>\n\n/' ${chart}/CHANGELOG.md
|
||||
rm ${chart}/app-changelog.md || echo "changelog not found..."
|
||||
fi
|
||||
}
|
||||
export -f create_changelog
|
||||
|
||||
generate_docs() {
|
||||
local chart="$1"
|
||||
local chartname="$2"
|
||||
local train="$3"
|
||||
local chartversion="$4"
|
||||
if [[ -z "$standalone" ]]; then
|
||||
echo "Generating Docs"
|
||||
if [ "${chartname}" == "common" ]; then
|
||||
helm-docs \
|
||||
--ignore-file=".helmdocsignore" \
|
||||
--output-file="README.md" \
|
||||
--template-files="/__w/apps/apps/templates/docs/common-README.md.gotmpl" \
|
||||
--chart-search-root="${chart}"
|
||||
helm-docs \
|
||||
--ignore-file=".helmdocsignore" \
|
||||
--output-file="helm-values.md" \
|
||||
--template-files="/__w/apps/apps/templates/docs/common-helm-values.md.gotmpl" \
|
||||
--chart-search-root="${chart}"
|
||||
else
|
||||
helm-docs \
|
||||
--ignore-file=".helmdocsignore" \
|
||||
--output-file="README.md" \
|
||||
--template-files="/__w/apps/apps/templates/docs/README.md.gotmpl" \
|
||||
--chart-search-root="${chart}"
|
||||
helm-docs \
|
||||
--ignore-file=".helmdocsignore" \
|
||||
--output-file="CONFIG.md" \
|
||||
--template-files="/__w/apps/apps/templates/docs/CONFIG.md.gotmpl" \
|
||||
--chart-search-root="${chart}"
|
||||
helm-docs \
|
||||
--ignore-file=".helmdocsignore" \
|
||||
--output-file="helm-values.md" \
|
||||
--template-files="/__w/apps/apps/templates/docs/helm-values.md.gotmpl" \
|
||||
--chart-search-root="${chart}"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
export -f generate_docs
|
||||
|
||||
parse_command_line() {
|
||||
while :; do
|
||||
case "${1:-}" in
|
||||
-h|--help)
|
||||
show_help
|
||||
exit
|
||||
;;
|
||||
-d|--charts-dir)
|
||||
if [[ -n "${2:-}" ]]; then
|
||||
charts_dir="$2"
|
||||
shift
|
||||
else
|
||||
echo "ERROR: '-d|--charts-dir' cannot be empty." >&2
|
||||
show_help
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
break
|
||||
;;
|
||||
esac
|
||||
|
||||
shift
|
||||
done
|
||||
}
|
||||
export -f parse_command_line
|
||||
|
||||
lookup_latest_tag() {
|
||||
git fetch --tags > /dev/null 2>&1
|
||||
|
||||
if ! git describe --tags --abbrev=0 2> /dev/null; then
|
||||
git rev-list --max-parents=0 --first-parent HEAD
|
||||
fi
|
||||
}
|
||||
export -f lookup_latest_tag
|
||||
|
||||
filter_charts() {
|
||||
while read -r chart; do
|
||||
[[ ! -d "$chart" ]] && continue
|
||||
if [[ $(git diff $latest_tag $chart/Chart.yaml | grep "+version") ]]; then
|
||||
echo "$chart"
|
||||
else
|
||||
echo "Version not bumped. Skipping." 1>&2
|
||||
fi
|
||||
done
|
||||
}
|
||||
export -f filter_charts
|
||||
|
||||
lookup_changed_charts() {
|
||||
local commit="$1"
|
||||
|
||||
local changed_files
|
||||
changed_files=$(git diff --find-renames --name-only "$commit" -- "$charts_dir" | grep "Chart.yaml")
|
||||
|
||||
local depth=$(( $(tr "/" "\n" <<< "$charts_dir" | sed '/^\(\.\)*$/d' | wc -l) + 1 ))
|
||||
local fields="1-${depth}"
|
||||
|
||||
cut -d '/' -f "$fields" <<< "$changed_files" | uniq | filter_charts
|
||||
}
|
||||
export -f lookup_changed_charts
|
||||
|
||||
main "$@"
|
||||
@@ -55,25 +55,7 @@ jobs:
|
||||
if: |
|
||||
steps.collect-changes.outputs.changesDetected == 'true'
|
||||
run: |
|
||||
# To be done, we should splitreadme generation from catalog generation
|
||||
# chart_runner(){
|
||||
# IFS='/' read -r -a chart_parts <<< "${1}"
|
||||
# if [ -f "charts/${chart_parts[0]}"/"${chart_parts[1]}/Chart.yaml" ]; then
|
||||
# ./.github/scripts/gen-helm-docs.sh "${chart_parts[0]}" "${chart_parts[1]}"
|
||||
# fi
|
||||
# }
|
||||
# export -f chart_runner
|
||||
# CHARTS=(${{ steps.collect-changes.outputs.addedOrModifiedCharts }})
|
||||
# parallel -j 8 chart_runner '2>&1' ::: ${CHARTS[@]}
|
||||
|
||||
- name: build catalogs and docs
|
||||
if: |
|
||||
steps.collect-changes.outputs.changesDetected == 'true'
|
||||
run: |
|
||||
pip install yq
|
||||
tools/build-release.sh -p
|
||||
env:
|
||||
CR_TOKEN: ${{ secrets.BOT_TOKEN }}
|
||||
.github/scripts/gen-docs.sh
|
||||
|
||||
- name: Collect changes
|
||||
id: collect-changes
|
||||
@@ -113,21 +95,6 @@ jobs:
|
||||
fi
|
||||
done
|
||||
|
||||
|
||||
- name: Commit Website Changes
|
||||
if: |
|
||||
steps.collect-changes.outputs.changesDetected == 'true'
|
||||
run: |
|
||||
cd pub
|
||||
git config user.name "TrueCharts-Bot"
|
||||
git config user.email "bot@truecharts.org"
|
||||
git add --all
|
||||
git commit -sm "Commit released docs for TrueCharts" || exit 0
|
||||
## TODO: actually push the content
|
||||
# git push
|
||||
cd -
|
||||
rm -rf pub
|
||||
|
||||
- name: Create commit
|
||||
id: create-commit
|
||||
uses: stefanzweifel/git-auto-commit-action@c4b132ec2c77a21fcab564bd3c92610cee84b894 # tag=v4
|
||||
@@ -147,6 +114,29 @@ jobs:
|
||||
echo "::set-output name=commit_hash::${GITHUB_SHA}"
|
||||
fi
|
||||
|
||||
- name: Commit Website Changes
|
||||
if: |
|
||||
steps.collect-changes.outputs.changesDetected == 'true'
|
||||
run: |
|
||||
cd pub
|
||||
git config user.name "TrueCharts-Bot"
|
||||
git config user.email "bot@truecharts.org"
|
||||
git add --all
|
||||
git commit -sm "Commit released docs for TrueCharts" || exit 0
|
||||
## TODO: actually push the content
|
||||
# git push
|
||||
cd -
|
||||
rm -rf pub
|
||||
|
||||
- name: build catalogs
|
||||
if: |
|
||||
steps.collect-changes.outputs.changesDetected == 'true'
|
||||
run: |
|
||||
pip install yq
|
||||
tools/build-release.sh -p
|
||||
env:
|
||||
CR_TOKEN: ${{ secrets.BOT_TOKEN }}
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846 # tag=v3
|
||||
with:
|
||||
|
||||
@@ -78,12 +78,12 @@ chart_runner(){
|
||||
train=$(basename $(dirname "${1}"))
|
||||
SCALESUPPORT=$(cat ${1}/Chart.yaml | yq '.annotations."truecharts.org/SCALE-support"' -r)
|
||||
helm dependency update "${1}" --skip-refresh || (sleep 10 && helm dependency update "${1}" --skip-refresh) || (sleep 10 && helm dependency update "${1}" --skip-refresh)
|
||||
helm_sec_scan "${1}" "${chartname}" "$train" "${chartversion}" || echo "helm-chart security-scan failed..."
|
||||
container_sec_scan "${1}" "${chartname}" "$train" "${chartversion}" || echo "container security-scan failed..."
|
||||
sec_scan_cleanup "${1}" "${chartname}" "$train" "${chartversion}" || echo "security-scan cleanup failed..."
|
||||
sync_tag "${1}" "${chartname}" "$train" "${chartversion}" || echo "Tag sync failed..."
|
||||
create_changelog "${1}" "${chartname}" "$train" "${chartversion}" || echo "changelog generation failed..."
|
||||
generate_docs "${1}" "${chartname}" "$train" "${chartversion}" || echo "Docs generation failed..."
|
||||
#helm_sec_scan "${1}" "${chartname}" "$train" "${chartversion}" || echo "helm-chart security-scan failed..."
|
||||
#container_sec_scan "${1}" "${chartname}" "$train" "${chartversion}" || echo "container security-scan failed..."
|
||||
#sec_scan_cleanup "${1}" "${chartname}" "$train" "${chartversion}" || echo "security-scan cleanup failed..."
|
||||
#sync_tag "${1}" "${chartname}" "$train" "${chartversion}" || echo "Tag sync failed..."
|
||||
#create_changelog "${1}" "${chartname}" "$train" "${chartversion}" || echo "changelog generation failed..."
|
||||
#generate_docs "${1}" "${chartname}" "$train" "${chartversion}" || echo "Docs generation failed..."
|
||||
#copy_docs "${1}" "${chartname}" "$train" "${chartversion}" || echo "Docs Copy failed..."
|
||||
#package_chart "${1}"
|
||||
if [[ "${SCALESUPPORT}" == "true" ]]; then
|
||||
|
||||
Reference in New Issue
Block a user