From 5af4f9b0e001fa8e0cf1e58a12091b9289bad92d Mon Sep 17 00:00:00 2001 From: Kjeld Schouten-Lebbing Date: Tue, 7 Sep 2021 20:37:52 +0200 Subject: [PATCH] Setup new test and release system (#919) * Tweak some release-build script parameters * adapt test to new release scripting * whoopsies * fix sh permissions * some fixes * use with * test bump * don't build indexes when running without production flag either * also validate the catalog * PWD/catalog * mtching " * Don't make change to primary repo when building catalog * Add doc generation feature * try fetch depth 100 * remove helm-prep step to speedup test init * also generate general docs * adapt release workflow and some workflow cleanup --- .github/workflows/apps.release.yaml | 133 ++-------------- .github/workflows/apps.test.yaml | 135 +--------------- .github/workflows/docs.deploy.yaml | 5 - charts/incubator/custom-app/Chart.yaml | 2 +- containers/apps/sogo/latest-version.sh | 0 .../apps/truecharts-release/latest-base.sh | 0 tools/build-release.sh | 148 ++++++++++++++---- tools/process.sh | 0 8 files changed, 142 insertions(+), 281 deletions(-) mode change 100644 => 100755 containers/apps/sogo/latest-version.sh mode change 100644 => 100755 containers/apps/truecharts-release/latest-base.sh mode change 100644 => 100755 tools/build-release.sh mode change 100644 => 100755 tools/process.sh diff --git a/.github/workflows/apps.release.yaml b/.github/workflows/apps.release.yaml index 66e51401572..e72c6ce9275 100644 --- a/.github/workflows/apps.release.yaml +++ b/.github/workflows/apps.release.yaml @@ -6,29 +6,14 @@ on: - 'master' tags-ignore: - '**' - paths: - - 'charts/**/chart.yaml' - - 'charts/**/Chart.yaml' - - '.github/workflows/apps.release.yaml' - - '.github/ct-install.yaml' - - '.github/ct-lint.yaml' - - '.github/cr.yaml' jobs: - app-release: - concurrency: gitpush + release: runs-on: ubuntu-latest + concurrency: gitpush container: image: ghcr.io/truecharts/truecharts-release:v1.0.0 steps: - # wait and Prep cache - - name: Block concurrent jobs - uses: softprops/turnstyle@v1 - with: - continue-after-seconds: 180 - env: - GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }} - - name: Cache helm repo cache id: cache uses: actions/cache@v2 @@ -38,127 +23,33 @@ jobs: ~/.cache/helm/repository ~/.config/helm/repositories.yaml - # Checkout repo's - - name: Checkout - uses: actions/checkout@v2 + - uses: actions/checkout@v2 + name: Checkout with: - fetch-depth: 0 - token: ${{ secrets.BOT_TOKEN }} - - name: Configure Git - run: | - git config user.name "TrueCharts-Bot" - git config user.email "bot@truecharts.org" - + fetch-depth: 100 - name: Checkout uses: actions/checkout@v2 with: - fetch-depth: 0 repository: truecharts/catalog - token: ${{ secrets.BOT_TOKEN }} path: catalog - - # Setup Helm - name: Install Helm uses: azure/setup-helm@v1 with: version: v3.5.3 - - name: install brew and chart releaser + - name: build-and-run run: | - cr --help + tools/build-release.sh --config .github/cr.yaml - - name: update helm repo cache + - name: Commit and Push new Helm Charts and docs run: | - helm repo add truecharts https://truecharts.org - helm repo add bitnami https://charts.bitnami.com/bitnami - helm repo update + git config user.name "TrueCharts-Bot" + git config user.email "bot@truecharts.org" + git add --all + git commit -sm "Commit released Helm Chart and docs for TrueCharts" || exit 0 - # HELM Release - - - name: Run chart-releaser - uses: helm/chart-releaser-action@v1.2.1 - with: - config: .github/cr.yaml - charts_dir: charts/* - env: - CR_TOKEN: "${{ secrets.BOT_TOKEN }}" - - # SCALE Release - - name: fetch dependencies - run: | - for train in stable incubator - do - for chart in charts/${train}/*; do - if [ -d "${chart}" ]; then - cd ${chart} - helm dependency update --skip-refresh - cd - - fi - done - done - - - name: Remove if release already exists - run: | - for train in stable incubator - do - for chart in charts/${train}/*; do - if [ -d "${chart}/SCALE" ]; then - maxchartversion=$(cat ${chart}/Chart.yaml | grep "^version: " | awk -F" " '{ print $2 }') - chartname=$(basename ${chart}) - echo "Processing... cleanup-existing... Chart: ${chartname} - Version: ${maxchartversion}" - rm -Rf catalog/${train}/${chartname}/${maxchartversion} || echo "Chart release ${chartname} with version ${maxchartversion} does not exist yet... Skipping..." - rm -Rf catalog/${train}/${chartname}/item.yaml || echo "No old catalog item.yaml found for ${chartname}" - else - echo "Skipping chart ${chart}, no correct SCALE compatibility layer detected" - fi - done - done - - - name: Apply SCALE Patches - run: | - for train in stable incubator - do - for chart in charts/${train}/*; do - if [ -d "${chart}/SCALE" ]; then - chartname=$(basename ${chart}) - echo "Processing... Applying SCALE patch... Chart: ${chartname} - Version: ${maxfolderversion}" - mv ${chart}/SCALE/item.yaml ${chart}/ - mv ${chart}/SCALE/ix_values.yaml ${chart}/ - mv ${chart}/SCALE/questions.yaml ${chart}/ - cp -rf ${chart}/SCALE/templates/* ${chart}/templates || echo "ignoring templates directory..." - rm -rf ${chart}/SCALE - mv ${chart}/values.yaml ${chart}/test_values.yaml - touch ${chart}/values.yaml - description=$(cat ${chart}/Chart.yaml | grep "^description: " | sed -r 's/^description: //') - echo "${description}" >> ${chart}/app-readme.md - else - echo "Skipping chart ${chart}, no correct SCALE compatibility layer detected" - fi - done - done - - name: Add Chart release - run: | - for train in stable incubator - do - for chart in charts/${train}/*; do - if [ -f "${chart}/item.yaml" ]; then - maxchartversion=$(cat ${chart}/Chart.yaml | grep "^version: " | awk -F" " '{ print $2 }') - chartname=$(basename ${chart}) - echo "Processing... Copying charts... Chart: ${chartname} - Version: ${maxchartversion}" - mkdir -p catalog/${train}/${chartname}/${maxchartversion} - cp -Rf ${chart}/* catalog/${train}/${chartname}/${maxchartversion}/ - mv catalog/${train}/${chartname}/${maxchartversion}/item.yaml catalog/${train}/${chartname}/item.yaml - else - echo "Skipping, chart ${chart} does not contain SCALE compatibility layer..." - fi - done - done - - name: Validate catalog format - run: | - cd catalog - /bin/bash -c "PWD=${pwd}; /usr/local/bin/catalog_validate validate --path $PWD" - name: Commit and Push new App releases run: | cd catalog diff --git a/.github/workflows/apps.test.yaml b/.github/workflows/apps.test.yaml index e3f5974387f..514912c1b6d 100644 --- a/.github/workflows/apps.test.yaml +++ b/.github/workflows/apps.test.yaml @@ -13,29 +13,6 @@ on: - '.github/ct-lint.yaml' jobs: - prephelm: - name: Prepare Helm - runs-on: ubuntu-20.04 - steps: - - name: Install Helm - uses: azure/setup-helm@v1 - with: - version: v3.5.3 - - - name: Cache helm repo cache - uses: actions/cache@v2 - with: - key: helmrepocache-${{ github.sha }} - path: | - ~/.cache/helm/repository - ~/.config/helm/repositories.yaml - - - name: update helm repo cache - run: | - helm repo add truecharts https://truecharts.org - helm repo add bitnami https://charts.bitnami.com/bitnami - helm repo update - changes: name: Get changed Apps runs-on: ubuntu-20.04 @@ -68,7 +45,7 @@ jobs: fi app-tests: - needs: [changes, prephelm] + needs: [changes] if: ${{ needs.changes.outputs.empty_matrix == 'false' }} name: App Tests runs-on: ubuntu-latest @@ -122,8 +99,7 @@ jobs: with: version: v3.5.3 - - name: Cache-Error-Fix - if: steps.cache.outputs.cache-hit != 'true' + - name: update helm repo cache run: | helm repo add truecharts https://truecharts.org helm repo add bitnami https://charts.bitnami.com/bitnami @@ -217,11 +193,10 @@ jobs: - name: complete message run: echo "App Tests Completed Successfully" - catalog-test: - needs: [prephelm] + release-test: runs-on: ubuntu-latest container: - image: ixsystems/catalog_validation:latest + image: ghcr.io/truecharts/truecharts-release:v1.0.0 steps: - name: Cache helm repo cache id: cache @@ -234,16 +209,12 @@ jobs: - uses: actions/checkout@v2 name: Checkout + with: + fetch-depth: 100 - name: Checkout uses: actions/checkout@v2 with: - fetch-depth: 0 - path: master - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: 0 repository: truecharts/catalog path: catalog @@ -252,96 +223,6 @@ jobs: with: version: v3.5.3 - - name: Cache-Error-Fix - if: steps.cache.outputs.cache-hit != 'true' + - name: build-and-run run: | - helm repo add truecharts https://truecharts.org - helm repo add bitnami https://charts.bitnami.com/bitnami - helm repo update - - - name: Remove if release already exists - run: | - removetask(){ - if [ -d "$2/SCALE" ]; then - maxchartversion=$(cat $2/Chart.yaml | grep "^version: " | awk -F" " '{ print $2 }') - chartname=$(basename $2) - echo "Processing... cleanup-existing... Chart: ${chartname} - Version: ${maxchartversion}" - rm -Rf catalog/$1/${chartname}/${maxchartversion} || echo "Chart release ${chartname} with version ${maxchartversion} does not exist yet... Skipping..." - rm -Rf catalog/$1/${chartname}/item.yaml || echo "No old catalog item.yaml found for ${chartname}" - else - echo "Skipping chart ${chart}, no correct SCALE compatibility layer detected" - fi - } - for train in stable incubator - do - for chart in master/charts/${train}/*; do - removetask "${train}" "${chart}" & - done - done - wait - - name: fetch dependencies - run: | - fetchtask(){ - if [ -d "$1" ]; then - cd $1 - helm dependency update --skip-refresh - cd - - fi - } - for train in stable incubator - do - for chart in master/charts/${train}/*; do - fetchtask "${chart}" - done - done - - name: Apply SCALE Patches - run: | - patchtask(){ - if [ -d "$1/SCALE" ]; then - chartname=$(basename $1) - echo "Processing... Applying SCALE patch... Chart: ${chartname}" - mv $1/SCALE/item.yaml $1/ - mv $1/SCALE/ix_values.yaml $1/ - mv $1/SCALE/questions.yaml $1/ - cp -rf $1/SCALE/templates/* $1/templates || echo "ignoring templates directory..." - rm -rf $1/SCALE - mv $1/values.yaml $1/test_values.yaml - touch $1/values.yaml - description=$(cat $1/Chart.yaml | grep "^description: " | sed -r 's/^description: //') - echo "${description}" >> $1/app-readme.md - else - echo "Skipping chart $1, no correct SCALE compatibility layer detected" - fi - } - for train in stable incubator - do - for chart in master/charts/${train}/*; do - patchtask "${chart}" & - done - done - wait - - name: Add Chart release - run: | - addtask(){ - if [ -f "$1/item.yaml" ]; then - maxchartversion=$(cat $1/Chart.yaml | grep "^version: " | awk -F" " '{ print $2 }') - chartname=$(basename $1) - echo "Processing... Copying charts... Chart: ${chartname} - Version: ${maxchartversion}" - mkdir -p catalog/${train}/${chartname}/${maxchartversion} - cp -Rf $1/* catalog/${train}/${chartname}/${maxchartversion}/ - mv catalog/${train}/${chartname}/${maxchartversion}/item.yaml catalog/${train}/${chartname}/item.yaml - else - echo "Skipping, chart $1 does not contain SCALE compatibility layer..." - fi - } - for train in stable incubator - do - for chart in master/charts/${train}/*; do - addtask "${chart}" & - done - done - wait - - name: Validate catalog format - run: | - cd catalog - /bin/bash -c "PWD=${pwd}; /usr/local/bin/catalog_validate validate --path $PWD" + tools/build-release.sh --config .github/cr.yaml diff --git a/.github/workflows/docs.deploy.yaml b/.github/workflows/docs.deploy.yaml index 01508996c59..74a9de83d51 100644 --- a/.github/workflows/docs.deploy.yaml +++ b/.github/workflows/docs.deploy.yaml @@ -7,7 +7,6 @@ on: paths: - 'docs/**' - '.github/workflows/docs.deploy.yaml' - - 'index.yaml' - 'mkdocs.yml' jobs: @@ -20,10 +19,6 @@ jobs: with: token: ${{ secrets.BOT_TOKEN }} - - name: Copy index.yaml to website - run: | - yes | cp -rf index.yaml docs/index.yaml || echo "chart-index copy failed, continuing..." - - name: Deploy docs uses: mhausenblas/mkdocs-deploy-gh-pages@master env: diff --git a/charts/incubator/custom-app/Chart.yaml b/charts/incubator/custom-app/Chart.yaml index 4ab41ff0883..7f0d1d0fa43 100644 --- a/charts/incubator/custom-app/Chart.yaml +++ b/charts/incubator/custom-app/Chart.yaml @@ -19,4 +19,4 @@ name: custom-app sources: - https://github.com/truecharts/apps/tree/master/charts/stable/custom-app type: application -version: 0.2.19 +version: 0.2.20 diff --git a/containers/apps/sogo/latest-version.sh b/containers/apps/sogo/latest-version.sh old mode 100644 new mode 100755 diff --git a/containers/apps/truecharts-release/latest-base.sh b/containers/apps/truecharts-release/latest-base.sh old mode 100644 new mode 100755 diff --git a/tools/build-release.sh b/tools/build-release.sh old mode 100644 new mode 100755 index 2a6d2b94405..68ac84f2ccb --- a/tools/build-release.sh +++ b/tools/build-release.sh @@ -31,22 +31,24 @@ Usage: $(basename "$0") -o, --owner The repo owner -r, --repo The repo name -p, --production Enables uploading releases to github releases + -s, --standalone Disables Chart Releaser and Catalog Validation, for local generation EOF } main() { local version="$DEFAULT_CHART_RELEASER_VERSION" local config= - local charts_dir=charts + local standalone= + local charts_dir=charts/** local owner= local repo= local production= local charts_repo_url= - local token=${CR_TOKEN:-none} + local token=${CR_TOKEN:-false} parse_command_line "$@" - if [ "${token}" == "none" ]; then - echo "env CR_TOKEN not found, defaulting to production=false" + if [ "${token}" == "false" ]; then + echo "env #cr_TOKEN not found, defaulting to production=false" production="false" fi @@ -61,9 +63,8 @@ main() { echo "Discovering changed charts since '$latest_tag'..." local changed_charts=() readarray -t changed_charts <<< "$(lookup_changed_charts "$latest_tag")" - + copy_general_docs if [[ -n "${changed_charts[*]}" ]]; then - install_chart_releaser rm -rf .cr-release-packages mkdir -p .cr-release-packages @@ -71,17 +72,20 @@ main() { rm -rf .cr-index mkdir -p .cr-index + prep_helm for chart in "${changed_charts[@]}"; do if [[ -d "$chart" ]]; then chartversion=$(cat ${chart}/Chart.yaml | grep "^version: " | awk -F" " '{ print $2 }') chartname=$(basename ${chart}) - prep_app "${chart}" + train=$(basename $(dirname "$chart")) + generate_docs "$chart" "$chartname" "$train" "$chartversion" + copy_docs "$chart" "$chartname" "$train" "$chartversion" helm dependency update "${chart}" --skip-refresh package_chart "$chart" if [[ -d "${chart}/SCALE" ]]; then clean_apps "$chart" "$chartname" "$train" "$chartversion" - patch_apps "$chart" "$chartname" "$train" "$chartversion" copy_apps "$chart" "$chartname" "$train" "$chartversion" + patch_apps "$chart" "$chartname" "$train" "$chartversion" else echo "Skipping chart ${chart}, no correct SCALE compatibility layer detected" fi @@ -89,10 +93,11 @@ main() { echo "Chart '$chart' no longer exists in repo. Skipping it..." fi done + validate_catalog if [ "${production}" == "true" ]; then release_charts - fi update_index + fi else echo "Nothing to do. No chart changes detected." fi @@ -100,12 +105,90 @@ main() { popd > /dev/null } -prep_app() { +copy_general_docs() { + yes | cp -rf index.yaml docs/index.yaml 2>/dev/null || : + yes | cp -rf .github/README.md docs/index.md 2>/dev/null || : + sed -i '1s/^/---\nhide:\n - navigation\n - toc\n---\n/' docs/index.md + sed -i 's~~~g' docs/index.md + yes | cp -rf .github/CODE_OF_CONDUCT docs/about/code_of_conduct.md 2>/dev/null || : + yes | cp -rf .github/CONTRIBUTING docs/development/contributing.md 2>/dev/null || : + yes | cp -rf .github/SUPPORT.md docs/manual/SUPPORT.md 2>/dev/null || : + yes | cp -rf LICENSE docs/about/legal/LICENSE.md 2>/dev/null || : + sed -i '1s/^/# License
\n\n/' docs/about/legal/LICENSE.md + yes | cp -rf NOTICE docs/about/legal/NOTICE.md 2>/dev/null || : + sed -i '1s/^/# NOTICE
\n\n/' docs/about/legal/NOTICE.md + } + +generate_docs() { local chart="$1" - echo "Creating app-readme.md for App: ${chartname}" - description=$(cat ${chart}/Chart.yaml | grep "^description: " | sed -r 's/^description: //') - echo "${description}" >> ${chart}/app-readme.md -} + local chartname="$2" + local train="$3" + local chartversion="$4" + if [[ -z "$standalone" ]]; then + echo "Generating Docs" + if [ "${chartname}" == "common" ]; then + helm-docs \ + --ignore-file=".helmdocsignore" \ + --output-file="README.md" \ + --template-files="/home/runner/work/apps/apps/templates/docs/common-README.md.gotmpl" \ + --chart-search-root="/home/runner/work/apps/apps/charts/library/common" + helm-docs \ + --ignore-file=".helmdocsignore" \ + --output-file="helm-values.md" \ + --template-files="/home/runner/work/apps/apps/templates/docs/common-helm-values.md.gotmpl" \ + --chart-search-root="/home/runner/work/apps/apps/charts/library/common" + else + helm-docs \ + --ignore-file=".helmdocsignore" \ + --output-file="README.md" \ + --template-files="/home/runner/work/apps/apps/templates/docs/README.md.gotmpl" \ + --chart-search-root="${chart}" + helm-docs \ + --ignore-file=".helmdocsignore" \ + --output-file="CONFIG.md" \ + --template-files="/home/runner/work/apps/apps/templates/docs/CONFIG.md.gotmpl" \ + --chart-search-root="${chart}" + helm-docs \ + --ignore-file=".helmdocsignore" \ + --output-file="app-readme.md" \ + --template-files="/home/runner/work/apps/apps/templates/docs/app-readme.md.gotmpl" \ + --chart-search-root="${chart}" + helm-docs \ + --ignore-file=".helmdocsignore" \ + --output-file="helm-values.md" \ + --template-files="/home/runner/work/apps/apps/templates/docs/helm-values.md.gotmpl" \ + --chart-search-root="${chart}" + fi + fi + } + + +copy_docs() { + local chart="$1" + local chartname="$2" + local train="$3" + local chartversion="$4" + echo "Copying docs for: ${chart}" + if [ "${chartname}" == "common" ]; then + mkdir -p docs/apps/common || : + yes | cp -rf charts/library/common/README.md docs/apps/common/index.md 2>/dev/null || : + yes | cp -rf charts/library/common/helm-values.md docs/apps/common/helm-values.md 2>/dev/null || : + else + mkdir -p docs/apps/${train}/${chartname} || echo "app path already exists, continuing..." + yes | cp -rf ${chart}/README.md docs/apps/${train}/${chartname}/index.md 2>/dev/null || : + yes | cp -rf ${chart}/CONFIG.md docs/apps/${train}/${chartname}/CONFIG.md 2>/dev/null || : + yes | cp -rf ${chart}/helm-values.md docs/apps/${train}/${chartname}/helm-values.md 2>/dev/null || : + sed -i '1s/^/# License
\n\n/' docs/apps/${train}/${chartname}/LICENSE.md 2>/dev/null || : + fi + } + +prep_helm() { + if [[ -z "$standalone" ]]; then + helm repo add truecharts https://truecharts.org + helm repo add bitnami https://charts.bitnami.com/bitnami + helm repo update + fi + } clean_apps() { local chart="$1" @@ -113,8 +196,8 @@ clean_apps() { local train="$3" local chartversion="$4" echo "Cleaning SCALE catalog for App: ${chartname}" - rm -Rf catalog/${train}/${chartname}/${chartversion} || true - rm -Rf catalog/${train}/${chartname}/item.yaml || true + rm -Rf catalog/${train}/${chartname}/${chartversion} 2>/dev/null || : + rm -Rf catalog/${train}/${chartname}/item.yaml 2>/dev/null || : } patch_apps() { @@ -122,14 +205,15 @@ patch_apps() { local chartname="$2" local train="$3" local chartversion="$4" + local target="catalog/${train}/${chartname}/${chartversion}" echo "Applying SCALE patches for App: ${chartname}" - mv ${chart}/SCALE/item.yaml ${chart}/ - mv ${chart}/SCALE/ix_values.yaml ${chart}/ - mv ${chart}/SCALE/questions.yaml ${chart}/ - cp -rf ${chart}/SCALE/templates/* ${chart}/templates || echo "ignoring templates directory..." - rm -rf ${chart}/SCALE - mv ${chart}/values.yaml ${chart}/test_values.yaml - touch ${chart}/values.yaml + mv ${target}/SCALE/ix_values.yaml ${target}/ + mv ${target}/SCALE/questions.yaml ${target}/ + cp -rf ${target}/SCALE/templates/* ${target}/templates 2>/dev/null || : + mv ${target}/SCALE/item.yaml catalog/${train}/${chartname}/item.yaml + rm -rf ${target}/SCALE + mv ${target}/values.yaml ${target}/test_values.yaml + touch ${target}/values.yaml } copy_apps() { @@ -140,12 +224,14 @@ copy_apps() { echo "Copying App to Catalog: ${2}" mkdir -p catalog/${train}/${chartname}/${chartversion} cp -Rf ${chart}/* catalog/${train}/${chartname}/${chartversion}/ - mv catalog/${train}/${chartname}/${chartversion}/item.yaml catalog/${train}/${chartname}/item.yaml + } validate_catalog() { + if [[ -z "$standalone" ]]; then echo "Starting Catalog Validation" - /bin/bash -c "PWD=${pwd}; /usr/local/bin/catalog_validate validate --path $PWD/catalog" + /usr/local/bin/catalog_validate validate --path "${PWD}/catalog" + fi } @@ -218,6 +304,9 @@ parse_command_line() { -p|--production) production="true" ;; + -s|--standalone) + standalone="true" + ;; *) break ;; @@ -265,7 +354,7 @@ lookup_changed_charts() { local commit="$1" local changed_files - changed_files=$(git diff --find-renames --name-only "$commit" -- "$charts_dir") + changed_files=$(git diff --find-renames --name-only "$commit" -- "$charts_dir" | grep "Chart.yaml") local depth=$(( $(tr "/" "\n" <<< "$charts_dir" | sed '/^\(\.\)*$/d' | wc -l) + 1 )) local fields="1-${depth}" @@ -275,14 +364,15 @@ lookup_changed_charts() { package_chart() { local chart="$1" - local args=("$chart" --package-path .cr-release-packages) if [[ -n "$config" ]]; then args+=(--config "$config") fi + if [[ -z "$standalone" ]]; then echo "Packaging chart '$chart'..." cr package "${args[@]}" + fi } release_charts() { @@ -291,8 +381,10 @@ release_charts() { args+=(--config "$config") fi + if [[ -z "$standalone" ]]; then echo 'Releasing charts...' cr upload "${args[@]}" + fi } update_index() { @@ -301,8 +393,10 @@ update_index() { args+=(--config "$config") fi + if [[ -z "$standalone" ]]; then echo 'Updating charts repo index...' cr index "${args[@]}" + fi } main "$@" diff --git a/tools/process.sh b/tools/process.sh old mode 100644 new mode 100755