Setup new test and release system (#919)

* Tweak some release-build script parameters

* adapt test to new release scripting

* whoopsies

* fix sh permissions

* some fixes

* use with

* test bump

* don't build indexes when running without production flag either

* also validate the catalog

* PWD/catalog

* mtching
"

* Don't make change to primary repo when building catalog

* Add doc generation feature

* try fetch depth 100

* remove helm-prep step to speedup test init

* also generate general docs

* adapt release workflow and some workflow cleanup
This commit is contained in:
Kjeld Schouten-Lebbing
2021-09-07 20:37:52 +02:00
committed by GitHub
parent f8bfaa4189
commit 5af4f9b0e0
8 changed files with 142 additions and 281 deletions
+12 -121
View File
@@ -6,29 +6,14 @@ on:
- 'master'
tags-ignore:
- '**'
paths:
- 'charts/**/chart.yaml'
- 'charts/**/Chart.yaml'
- '.github/workflows/apps.release.yaml'
- '.github/ct-install.yaml'
- '.github/ct-lint.yaml'
- '.github/cr.yaml'
jobs:
app-release:
concurrency: gitpush
release:
runs-on: ubuntu-latest
concurrency: gitpush
container:
image: ghcr.io/truecharts/truecharts-release:v1.0.0
steps:
# wait and Prep cache
- name: Block concurrent jobs
uses: softprops/turnstyle@v1
with:
continue-after-seconds: 180
env:
GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }}
- name: Cache helm repo cache
id: cache
uses: actions/cache@v2
@@ -38,127 +23,33 @@ jobs:
~/.cache/helm/repository
~/.config/helm/repositories.yaml
# Checkout repo's
- name: Checkout
uses: actions/checkout@v2
- uses: actions/checkout@v2
name: Checkout
with:
fetch-depth: 0
token: ${{ secrets.BOT_TOKEN }}
- name: Configure Git
run: |
git config user.name "TrueCharts-Bot"
git config user.email "bot@truecharts.org"
fetch-depth: 100
- name: Checkout
uses: actions/checkout@v2
with:
fetch-depth: 0
repository: truecharts/catalog
token: ${{ secrets.BOT_TOKEN }}
path: catalog
# Setup Helm
- name: Install Helm
uses: azure/setup-helm@v1
with:
version: v3.5.3
- name: install brew and chart releaser
- name: build-and-run
run: |
cr --help
tools/build-release.sh --config .github/cr.yaml
- name: update helm repo cache
- name: Commit and Push new Helm Charts and docs
run: |
helm repo add truecharts https://truecharts.org
helm repo add bitnami https://charts.bitnami.com/bitnami
helm repo update
git config user.name "TrueCharts-Bot"
git config user.email "bot@truecharts.org"
git add --all
git commit -sm "Commit released Helm Chart and docs for TrueCharts" || exit 0
# HELM Release
- name: Run chart-releaser
uses: helm/chart-releaser-action@v1.2.1
with:
config: .github/cr.yaml
charts_dir: charts/*
env:
CR_TOKEN: "${{ secrets.BOT_TOKEN }}"
# SCALE Release
- name: fetch dependencies
run: |
for train in stable incubator
do
for chart in charts/${train}/*; do
if [ -d "${chart}" ]; then
cd ${chart}
helm dependency update --skip-refresh
cd -
fi
done
done
- name: Remove if release already exists
run: |
for train in stable incubator
do
for chart in charts/${train}/*; do
if [ -d "${chart}/SCALE" ]; then
maxchartversion=$(cat ${chart}/Chart.yaml | grep "^version: " | awk -F" " '{ print $2 }')
chartname=$(basename ${chart})
echo "Processing... cleanup-existing... Chart: ${chartname} - Version: ${maxchartversion}"
rm -Rf catalog/${train}/${chartname}/${maxchartversion} || echo "Chart release ${chartname} with version ${maxchartversion} does not exist yet... Skipping..."
rm -Rf catalog/${train}/${chartname}/item.yaml || echo "No old catalog item.yaml found for ${chartname}"
else
echo "Skipping chart ${chart}, no correct SCALE compatibility layer detected"
fi
done
done
- name: Apply SCALE Patches
run: |
for train in stable incubator
do
for chart in charts/${train}/*; do
if [ -d "${chart}/SCALE" ]; then
chartname=$(basename ${chart})
echo "Processing... Applying SCALE patch... Chart: ${chartname} - Version: ${maxfolderversion}"
mv ${chart}/SCALE/item.yaml ${chart}/
mv ${chart}/SCALE/ix_values.yaml ${chart}/
mv ${chart}/SCALE/questions.yaml ${chart}/
cp -rf ${chart}/SCALE/templates/* ${chart}/templates || echo "ignoring templates directory..."
rm -rf ${chart}/SCALE
mv ${chart}/values.yaml ${chart}/test_values.yaml
touch ${chart}/values.yaml
description=$(cat ${chart}/Chart.yaml | grep "^description: " | sed -r 's/^description: //')
echo "${description}" >> ${chart}/app-readme.md
else
echo "Skipping chart ${chart}, no correct SCALE compatibility layer detected"
fi
done
done
- name: Add Chart release
run: |
for train in stable incubator
do
for chart in charts/${train}/*; do
if [ -f "${chart}/item.yaml" ]; then
maxchartversion=$(cat ${chart}/Chart.yaml | grep "^version: " | awk -F" " '{ print $2 }')
chartname=$(basename ${chart})
echo "Processing... Copying charts... Chart: ${chartname} - Version: ${maxchartversion}"
mkdir -p catalog/${train}/${chartname}/${maxchartversion}
cp -Rf ${chart}/* catalog/${train}/${chartname}/${maxchartversion}/
mv catalog/${train}/${chartname}/${maxchartversion}/item.yaml catalog/${train}/${chartname}/item.yaml
else
echo "Skipping, chart ${chart} does not contain SCALE compatibility layer..."
fi
done
done
- name: Validate catalog format
run: |
cd catalog
/bin/bash -c "PWD=${pwd}; /usr/local/bin/catalog_validate validate --path $PWD"
- name: Commit and Push new App releases
run: |
cd catalog
+8 -127
View File
@@ -13,29 +13,6 @@ on:
- '.github/ct-lint.yaml'
jobs:
prephelm:
name: Prepare Helm
runs-on: ubuntu-20.04
steps:
- name: Install Helm
uses: azure/setup-helm@v1
with:
version: v3.5.3
- name: Cache helm repo cache
uses: actions/cache@v2
with:
key: helmrepocache-${{ github.sha }}
path: |
~/.cache/helm/repository
~/.config/helm/repositories.yaml
- name: update helm repo cache
run: |
helm repo add truecharts https://truecharts.org
helm repo add bitnami https://charts.bitnami.com/bitnami
helm repo update
changes:
name: Get changed Apps
runs-on: ubuntu-20.04
@@ -68,7 +45,7 @@ jobs:
fi
app-tests:
needs: [changes, prephelm]
needs: [changes]
if: ${{ needs.changes.outputs.empty_matrix == 'false' }}
name: App Tests
runs-on: ubuntu-latest
@@ -122,8 +99,7 @@ jobs:
with:
version: v3.5.3
- name: Cache-Error-Fix
if: steps.cache.outputs.cache-hit != 'true'
- name: update helm repo cache
run: |
helm repo add truecharts https://truecharts.org
helm repo add bitnami https://charts.bitnami.com/bitnami
@@ -217,11 +193,10 @@ jobs:
- name: complete message
run: echo "App Tests Completed Successfully"
catalog-test:
needs: [prephelm]
release-test:
runs-on: ubuntu-latest
container:
image: ixsystems/catalog_validation:latest
image: ghcr.io/truecharts/truecharts-release:v1.0.0
steps:
- name: Cache helm repo cache
id: cache
@@ -234,16 +209,12 @@ jobs:
- uses: actions/checkout@v2
name: Checkout
with:
fetch-depth: 100
- name: Checkout
uses: actions/checkout@v2
with:
fetch-depth: 0
path: master
- name: Checkout
uses: actions/checkout@v2
with:
fetch-depth: 0
repository: truecharts/catalog
path: catalog
@@ -252,96 +223,6 @@ jobs:
with:
version: v3.5.3
- name: Cache-Error-Fix
if: steps.cache.outputs.cache-hit != 'true'
- name: build-and-run
run: |
helm repo add truecharts https://truecharts.org
helm repo add bitnami https://charts.bitnami.com/bitnami
helm repo update
- name: Remove if release already exists
run: |
removetask(){
if [ -d "$2/SCALE" ]; then
maxchartversion=$(cat $2/Chart.yaml | grep "^version: " | awk -F" " '{ print $2 }')
chartname=$(basename $2)
echo "Processing... cleanup-existing... Chart: ${chartname} - Version: ${maxchartversion}"
rm -Rf catalog/$1/${chartname}/${maxchartversion} || echo "Chart release ${chartname} with version ${maxchartversion} does not exist yet... Skipping..."
rm -Rf catalog/$1/${chartname}/item.yaml || echo "No old catalog item.yaml found for ${chartname}"
else
echo "Skipping chart ${chart}, no correct SCALE compatibility layer detected"
fi
}
for train in stable incubator
do
for chart in master/charts/${train}/*; do
removetask "${train}" "${chart}" &
done
done
wait
- name: fetch dependencies
run: |
fetchtask(){
if [ -d "$1" ]; then
cd $1
helm dependency update --skip-refresh
cd -
fi
}
for train in stable incubator
do
for chart in master/charts/${train}/*; do
fetchtask "${chart}"
done
done
- name: Apply SCALE Patches
run: |
patchtask(){
if [ -d "$1/SCALE" ]; then
chartname=$(basename $1)
echo "Processing... Applying SCALE patch... Chart: ${chartname}"
mv $1/SCALE/item.yaml $1/
mv $1/SCALE/ix_values.yaml $1/
mv $1/SCALE/questions.yaml $1/
cp -rf $1/SCALE/templates/* $1/templates || echo "ignoring templates directory..."
rm -rf $1/SCALE
mv $1/values.yaml $1/test_values.yaml
touch $1/values.yaml
description=$(cat $1/Chart.yaml | grep "^description: " | sed -r 's/^description: //')
echo "${description}" >> $1/app-readme.md
else
echo "Skipping chart $1, no correct SCALE compatibility layer detected"
fi
}
for train in stable incubator
do
for chart in master/charts/${train}/*; do
patchtask "${chart}" &
done
done
wait
- name: Add Chart release
run: |
addtask(){
if [ -f "$1/item.yaml" ]; then
maxchartversion=$(cat $1/Chart.yaml | grep "^version: " | awk -F" " '{ print $2 }')
chartname=$(basename $1)
echo "Processing... Copying charts... Chart: ${chartname} - Version: ${maxchartversion}"
mkdir -p catalog/${train}/${chartname}/${maxchartversion}
cp -Rf $1/* catalog/${train}/${chartname}/${maxchartversion}/
mv catalog/${train}/${chartname}/${maxchartversion}/item.yaml catalog/${train}/${chartname}/item.yaml
else
echo "Skipping, chart $1 does not contain SCALE compatibility layer..."
fi
}
for train in stable incubator
do
for chart in master/charts/${train}/*; do
addtask "${chart}" &
done
done
wait
- name: Validate catalog format
run: |
cd catalog
/bin/bash -c "PWD=${pwd}; /usr/local/bin/catalog_validate validate --path $PWD"
tools/build-release.sh --config .github/cr.yaml
-5
View File
@@ -7,7 +7,6 @@ on:
paths:
- 'docs/**'
- '.github/workflows/docs.deploy.yaml'
- 'index.yaml'
- 'mkdocs.yml'
jobs:
@@ -20,10 +19,6 @@ jobs:
with:
token: ${{ secrets.BOT_TOKEN }}
- name: Copy index.yaml to website
run: |
yes | cp -rf index.yaml docs/index.yaml || echo "chart-index copy failed, continuing..."
- name: Deploy docs
uses: mhausenblas/mkdocs-deploy-gh-pages@master
env:
+1 -1
View File
@@ -19,4 +19,4 @@ name: custom-app
sources:
- https://github.com/truecharts/apps/tree/master/charts/stable/custom-app
type: application
version: 0.2.19
version: 0.2.20
View File
View File
Regular → Executable
+121 -27
View File
@@ -31,22 +31,24 @@ Usage: $(basename "$0") <options>
-o, --owner The repo owner
-r, --repo The repo name
-p, --production Enables uploading releases to github releases
-s, --standalone Disables Chart Releaser and Catalog Validation, for local generation
EOF
}
main() {
local version="$DEFAULT_CHART_RELEASER_VERSION"
local config=
local charts_dir=charts
local standalone=
local charts_dir=charts/**
local owner=
local repo=
local production=
local charts_repo_url=
local token=${CR_TOKEN:-none}
local token=${CR_TOKEN:-false}
parse_command_line "$@"
if [ "${token}" == "none" ]; then
echo "env CR_TOKEN not found, defaulting to production=false"
if [ "${token}" == "false" ]; then
echo "env #cr_TOKEN not found, defaulting to production=false"
production="false"
fi
@@ -61,9 +63,8 @@ main() {
echo "Discovering changed charts since '$latest_tag'..."
local changed_charts=()
readarray -t changed_charts <<< "$(lookup_changed_charts "$latest_tag")"
copy_general_docs
if [[ -n "${changed_charts[*]}" ]]; then
install_chart_releaser
rm -rf .cr-release-packages
mkdir -p .cr-release-packages
@@ -71,17 +72,20 @@ main() {
rm -rf .cr-index
mkdir -p .cr-index
prep_helm
for chart in "${changed_charts[@]}"; do
if [[ -d "$chart" ]]; then
chartversion=$(cat ${chart}/Chart.yaml | grep "^version: " | awk -F" " '{ print $2 }')
chartname=$(basename ${chart})
prep_app "${chart}"
train=$(basename $(dirname "$chart"))
generate_docs "$chart" "$chartname" "$train" "$chartversion"
copy_docs "$chart" "$chartname" "$train" "$chartversion"
helm dependency update "${chart}" --skip-refresh
package_chart "$chart"
if [[ -d "${chart}/SCALE" ]]; then
clean_apps "$chart" "$chartname" "$train" "$chartversion"
patch_apps "$chart" "$chartname" "$train" "$chartversion"
copy_apps "$chart" "$chartname" "$train" "$chartversion"
patch_apps "$chart" "$chartname" "$train" "$chartversion"
else
echo "Skipping chart ${chart}, no correct SCALE compatibility layer detected"
fi
@@ -89,10 +93,11 @@ main() {
echo "Chart '$chart' no longer exists in repo. Skipping it..."
fi
done
validate_catalog
if [ "${production}" == "true" ]; then
release_charts
fi
update_index
fi
else
echo "Nothing to do. No chart changes detected."
fi
@@ -100,12 +105,90 @@ main() {
popd > /dev/null
}
prep_app() {
copy_general_docs() {
yes | cp -rf index.yaml docs/index.yaml 2>/dev/null || :
yes | cp -rf .github/README.md docs/index.md 2>/dev/null || :
sed -i '1s/^/---\nhide:\n - navigation\n - toc\n---\n/' docs/index.md
sed -i 's~<!-- INSERT-DISCORD-WIDGET -->~<iframe src="https://discord.com/widget?id=830763548678291466\&theme=dark" width="350" height="500" allowtransparency="true" frameborder="0" sandbox="allow-popups allow-popups-to-escape-sandbox allow-same-origin allow-scripts"></iframe>~g' docs/index.md
yes | cp -rf .github/CODE_OF_CONDUCT docs/about/code_of_conduct.md 2>/dev/null || :
yes | cp -rf .github/CONTRIBUTING docs/development/contributing.md 2>/dev/null || :
yes | cp -rf .github/SUPPORT.md docs/manual/SUPPORT.md 2>/dev/null || :
yes | cp -rf LICENSE docs/about/legal/LICENSE.md 2>/dev/null || :
sed -i '1s/^/# License<br>\n\n/' docs/about/legal/LICENSE.md
yes | cp -rf NOTICE docs/about/legal/NOTICE.md 2>/dev/null || :
sed -i '1s/^/# NOTICE<br>\n\n/' docs/about/legal/NOTICE.md
}
generate_docs() {
local chart="$1"
echo "Creating app-readme.md for App: ${chartname}"
description=$(cat ${chart}/Chart.yaml | grep "^description: " | sed -r 's/^description: //')
echo "${description}" >> ${chart}/app-readme.md
}
local chartname="$2"
local train="$3"
local chartversion="$4"
if [[ -z "$standalone" ]]; then
echo "Generating Docs"
if [ "${chartname}" == "common" ]; then
helm-docs \
--ignore-file=".helmdocsignore" \
--output-file="README.md" \
--template-files="/home/runner/work/apps/apps/templates/docs/common-README.md.gotmpl" \
--chart-search-root="/home/runner/work/apps/apps/charts/library/common"
helm-docs \
--ignore-file=".helmdocsignore" \
--output-file="helm-values.md" \
--template-files="/home/runner/work/apps/apps/templates/docs/common-helm-values.md.gotmpl" \
--chart-search-root="/home/runner/work/apps/apps/charts/library/common"
else
helm-docs \
--ignore-file=".helmdocsignore" \
--output-file="README.md" \
--template-files="/home/runner/work/apps/apps/templates/docs/README.md.gotmpl" \
--chart-search-root="${chart}"
helm-docs \
--ignore-file=".helmdocsignore" \
--output-file="CONFIG.md" \
--template-files="/home/runner/work/apps/apps/templates/docs/CONFIG.md.gotmpl" \
--chart-search-root="${chart}"
helm-docs \
--ignore-file=".helmdocsignore" \
--output-file="app-readme.md" \
--template-files="/home/runner/work/apps/apps/templates/docs/app-readme.md.gotmpl" \
--chart-search-root="${chart}"
helm-docs \
--ignore-file=".helmdocsignore" \
--output-file="helm-values.md" \
--template-files="/home/runner/work/apps/apps/templates/docs/helm-values.md.gotmpl" \
--chart-search-root="${chart}"
fi
fi
}
copy_docs() {
local chart="$1"
local chartname="$2"
local train="$3"
local chartversion="$4"
echo "Copying docs for: ${chart}"
if [ "${chartname}" == "common" ]; then
mkdir -p docs/apps/common || :
yes | cp -rf charts/library/common/README.md docs/apps/common/index.md 2>/dev/null || :
yes | cp -rf charts/library/common/helm-values.md docs/apps/common/helm-values.md 2>/dev/null || :
else
mkdir -p docs/apps/${train}/${chartname} || echo "app path already exists, continuing..."
yes | cp -rf ${chart}/README.md docs/apps/${train}/${chartname}/index.md 2>/dev/null || :
yes | cp -rf ${chart}/CONFIG.md docs/apps/${train}/${chartname}/CONFIG.md 2>/dev/null || :
yes | cp -rf ${chart}/helm-values.md docs/apps/${train}/${chartname}/helm-values.md 2>/dev/null || :
sed -i '1s/^/# License<br>\n\n/' docs/apps/${train}/${chartname}/LICENSE.md 2>/dev/null || :
fi
}
prep_helm() {
if [[ -z "$standalone" ]]; then
helm repo add truecharts https://truecharts.org
helm repo add bitnami https://charts.bitnami.com/bitnami
helm repo update
fi
}
clean_apps() {
local chart="$1"
@@ -113,8 +196,8 @@ clean_apps() {
local train="$3"
local chartversion="$4"
echo "Cleaning SCALE catalog for App: ${chartname}"
rm -Rf catalog/${train}/${chartname}/${chartversion} || true
rm -Rf catalog/${train}/${chartname}/item.yaml || true
rm -Rf catalog/${train}/${chartname}/${chartversion} 2>/dev/null || :
rm -Rf catalog/${train}/${chartname}/item.yaml 2>/dev/null || :
}
patch_apps() {
@@ -122,14 +205,15 @@ patch_apps() {
local chartname="$2"
local train="$3"
local chartversion="$4"
local target="catalog/${train}/${chartname}/${chartversion}"
echo "Applying SCALE patches for App: ${chartname}"
mv ${chart}/SCALE/item.yaml ${chart}/
mv ${chart}/SCALE/ix_values.yaml ${chart}/
mv ${chart}/SCALE/questions.yaml ${chart}/
cp -rf ${chart}/SCALE/templates/* ${chart}/templates || echo "ignoring templates directory..."
rm -rf ${chart}/SCALE
mv ${chart}/values.yaml ${chart}/test_values.yaml
touch ${chart}/values.yaml
mv ${target}/SCALE/ix_values.yaml ${target}/
mv ${target}/SCALE/questions.yaml ${target}/
cp -rf ${target}/SCALE/templates/* ${target}/templates 2>/dev/null || :
mv ${target}/SCALE/item.yaml catalog/${train}/${chartname}/item.yaml
rm -rf ${target}/SCALE
mv ${target}/values.yaml ${target}/test_values.yaml
touch ${target}/values.yaml
}
copy_apps() {
@@ -140,12 +224,14 @@ copy_apps() {
echo "Copying App to Catalog: ${2}"
mkdir -p catalog/${train}/${chartname}/${chartversion}
cp -Rf ${chart}/* catalog/${train}/${chartname}/${chartversion}/
mv catalog/${train}/${chartname}/${chartversion}/item.yaml catalog/${train}/${chartname}/item.yaml
}
validate_catalog() {
if [[ -z "$standalone" ]]; then
echo "Starting Catalog Validation"
/bin/bash -c "PWD=${pwd}; /usr/local/bin/catalog_validate validate --path $PWD/catalog"
/usr/local/bin/catalog_validate validate --path "${PWD}/catalog"
fi
}
@@ -218,6 +304,9 @@ parse_command_line() {
-p|--production)
production="true"
;;
-s|--standalone)
standalone="true"
;;
*)
break
;;
@@ -265,7 +354,7 @@ lookup_changed_charts() {
local commit="$1"
local changed_files
changed_files=$(git diff --find-renames --name-only "$commit" -- "$charts_dir")
changed_files=$(git diff --find-renames --name-only "$commit" -- "$charts_dir" | grep "Chart.yaml")
local depth=$(( $(tr "/" "\n" <<< "$charts_dir" | sed '/^\(\.\)*$/d' | wc -l) + 1 ))
local fields="1-${depth}"
@@ -275,14 +364,15 @@ lookup_changed_charts() {
package_chart() {
local chart="$1"
local args=("$chart" --package-path .cr-release-packages)
if [[ -n "$config" ]]; then
args+=(--config "$config")
fi
if [[ -z "$standalone" ]]; then
echo "Packaging chart '$chart'..."
cr package "${args[@]}"
fi
}
release_charts() {
@@ -291,8 +381,10 @@ release_charts() {
args+=(--config "$config")
fi
if [[ -z "$standalone" ]]; then
echo 'Releasing charts...'
cr upload "${args[@]}"
fi
}
update_index() {
@@ -301,8 +393,10 @@ update_index() {
args+=(--config "$config")
fi
if [[ -z "$standalone" ]]; then
echo 'Updating charts repo index...'
cr index "${args[@]}"
fi
}
main "$@"
Regular → Executable
View File