From d83d9f2e1856f615b242aac16d8d4a702d7e0b49 Mon Sep 17 00:00:00 2001 From: Kjeld Schouten-Lebbing Date: Mon, 16 Aug 2021 15:27:42 +0200 Subject: [PATCH] Housekeeping workflow rework (#771) * remove the chart version from readme template * only (re)generate docs once a day or on demand, not on every change * simplify workflow a bit * streamline docs generation some more * integrate helm-docs generation into to workflow * Turn it into a single daily housekeeping job * always include index.yaml when deploying docs... --- .github/workflows/containers.update.yaml | 61 -------- .github/workflows/docs.copy.yaml | 86 ----------- .github/workflows/docs.deploy.yaml | 5 + .github/workflows/housekeeping.yaml | 181 +++++++++++++++++++++++ tools/gen-helm-docs.sh | 39 ----- 5 files changed, 186 insertions(+), 186 deletions(-) delete mode 100644 .github/workflows/containers.update.yaml delete mode 100644 .github/workflows/docs.copy.yaml create mode 100644 .github/workflows/housekeeping.yaml delete mode 100755 tools/gen-helm-docs.sh diff --git a/.github/workflows/containers.update.yaml b/.github/workflows/containers.update.yaml deleted file mode 100644 index a9fc856d46d..00000000000 --- a/.github/workflows/containers.update.yaml +++ /dev/null @@ -1,61 +0,0 @@ -name: "Containers: Update" - -on: - workflow_dispatch: - push: - branches: - - main - paths: - - ".github/workflows/containers.update.yaml" - schedule: - - cron: "0 3 * * *" - -jobs: - get-versions: - name: Get Apps Versions - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - with: - token: ${{ secrets.BOT_TOKEN }} - fetch-depth: 1 - - name: Fetch new application versions - run: | - find ./containers/apps -maxdepth 1 -mindepth 1 -type d -exec basename {} \; | while read app; do - if test -f "./containers/apps/${app}/latest-version.sh"; then - version=$(bash "./containers/apps/${app}/latest-version.sh") - if [[ ! -z "${version}" || "${version}" != "null" ]]; then - echo "${version}" | tee "./containers/apps/${app}/VERSION" > /dev/null - echo "App: ${app} using version: ${version}" - fi - fi - if test -f "./containers/apps/${app}/BASE"; then - if test -f "./containers/apps/${app}/latest-base.sh"; then - base=$(bash "./containers/apps/${app}/latest-base.sh") - if [[ ! -z "${base}" || "${base}" != "null" ]]; then - echo "${base}" | tee "containers/apps/${app}/BASE" > /dev/null - echo "App: ${app} using Base: ${base}" - fi - fi - fi - done - - name: Fetch new base versions - run: | - find ./containers/base -maxdepth 1 -mindepth 1 -type d -exec basename {} \; | while read app; do - if test -f "./containers/base/${app}/latest-version.sh"; then - version=$(bash "./containers/base/${app}/latest-version.sh") - if [[ ! -z "${version}" || "${version}" != "null" ]]; then - echo "${version}" | tee "./containers/base/${app}/VERSION" > /dev/null - echo "${app} ${version}" - fi - fi - done - - name: Add, commit and push to applications version file - run: | - if [[ -n $(git status --porcelain) ]]; then - git config user.name "TrueCharts Bot" - git config user.email "bot@truecharts.org" - git add -A - git commit -sam "chore: added new container release versions" || exit 0 - git push - fi diff --git a/.github/workflows/docs.copy.yaml b/.github/workflows/docs.copy.yaml deleted file mode 100644 index 322012da02b..00000000000 --- a/.github/workflows/docs.copy.yaml +++ /dev/null @@ -1,86 +0,0 @@ -name: "Docs: Copy" - -on: - workflow_dispatch: - push: - branches: - - master - tags-ignore: - - '**' - paths: - - 'library/**/Chart.yaml' - - 'tools/gen-helm-docs.sh' - - 'tools/templates/docs/**' - - 'tools/templates/app/*.gotmpl' - - '**.gotmpl' - - '**/LICENSE' - - '**LICENSE' - - '**.md' - - '**/*.md' - - 'charts/**/*.md' - - '.github/README.md' - - '.github/CODE_OF_CONDUCT' - - '.github/CONTRIBUTING' - - './LICENSE' - - './NOTICE' - - '.github/workflows/docs.copy.yaml' - - 'index.yaml' - -jobs: - copy: - runs-on: ubuntu-latest - name: "Copy" - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - token: ${{ secrets.BOT_TOKEN }} - - - name: install helm-docs - run: | - brew install norwoodj/tap/helm-docs - -# - name: (re)generate docs -# run: | -# #tools/gen-helm-docs.sh - - - name: Copy general readme to website - run: | - yes | cp -rf index.yaml docs/index.yaml || echo "chart-index copy failed, continuing..." - yes | cp -rf .github/README.md docs/index.md || echo "readme copy failed, continuing..." - sed -i '1s/^/---\nhide:\n - navigation\n - toc\n---\n/' docs/index.md - sed -i 's~~~g' docs/index.md - yes | cp -rf .github/CODE_OF_CONDUCT docs/about/code_of_conduct.md || echo "CODE_OF_CONDUCT copy failed, continuing..." - yes | cp -rf .github/CONTRIBUTING docs/development/contributing.md || echo "CONTRIBUTING copy failed, continuing..." - yes | cp -rf .github/SUPPORT.md docs/manual/SUPPORT.md || echo "support policy copy failed, continuing..." - yes | cp -rf LICENSE docs/about/legal/LICENSE.md || echo "license copy failed, continuing..." - sed -i '1s/^/# License
\n\n/' docs/about/legal/LICENSE.md - yes | cp -rf NOTICE docs/about/legal/NOTICE.md || echo "license copy failed, continuing..." - sed -i '1s/^/# NOTICE
\n\n/' docs/about/legal/NOTICE.md - ls docs/ - - - name: Copy Apps readme to website - run: | - for train in stable incubator develop non-free deprecated; do - for chart in ${train}/*; do - if [ -d "${chart}" ]; then - chartname=$(basename ${chart}) - echo "Processing: ${chart}" - mkdir -p docs/apps/${train}/${chartname} || echo "app path already exists, continuing..." - yes | cp -rf ${chart}/README.md docs/apps/${train}/${chartname}/index.md || echo "readme copy failed, continuing..." - yes | cp -rf ${chart}/CONFIG.md docs/apps/${train}/${chartname}/CONFIG.md || echo "config copy failed, continuing..." - yes | cp -rf ${chart}/LICENSE docs/apps/${train}/${chartname}/LICENSE.md || echo "license copy failed, continuing..." - sed -i '1s/^/# License
\n\n/' docs/apps/${train}/${chartname}/LICENSE.md || echo "license edit failed, continuing..." - fi - done - echo "Processed charts for ${train}:" - ls docs/apps/${train}/ || echo "no Apps in train" - done - - - name: Commit and Push updated docs - run: | - git config user.name "TrueCharts-Bot" - git config user.email "bot@truecharts.org" - git add --all - git commit -sm "Commit docs updates" || exit 0 - git push diff --git a/.github/workflows/docs.deploy.yaml b/.github/workflows/docs.deploy.yaml index 74a9de83d51..01508996c59 100644 --- a/.github/workflows/docs.deploy.yaml +++ b/.github/workflows/docs.deploy.yaml @@ -7,6 +7,7 @@ on: paths: - 'docs/**' - '.github/workflows/docs.deploy.yaml' + - 'index.yaml' - 'mkdocs.yml' jobs: @@ -19,6 +20,10 @@ jobs: with: token: ${{ secrets.BOT_TOKEN }} + - name: Copy index.yaml to website + run: | + yes | cp -rf index.yaml docs/index.yaml || echo "chart-index copy failed, continuing..." + - name: Deploy docs uses: mhausenblas/mkdocs-deploy-gh-pages@master env: diff --git a/.github/workflows/housekeeping.yaml b/.github/workflows/housekeeping.yaml new file mode 100644 index 00000000000..8c6d1858dd4 --- /dev/null +++ b/.github/workflows/housekeeping.yaml @@ -0,0 +1,181 @@ +name: "Housekeeping" + +on: + workflow_dispatch: + push: + branches: + - main + paths: + - ".github/workflows/housekeeping.yaml" + schedule: + - cron: "0 3 * * *" + +jobs: + housekeeping: + runs-on: ubuntu-latest + name: "Docs: (re)Generate" + steps: + - name: Checkout branch + uses: actions/checkout@v2 + with: + fetch-depth: 0 + token: ${{ secrets.BOT_TOKEN }} + + - name: Set up Python + uses: actions/setup-python@v2 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pybump + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + + - name: install helm-docs + run: | + brew install norwoodj/tap/helm-docs + + - name: Containers: Fetch new application versions + run: | + find ./containers/apps -maxdepth 1 -mindepth 1 -type d -exec basename {} \; | while read app; do + if test -f "./containers/apps/${app}/latest-version.sh"; then + version=$(bash "./containers/apps/${app}/latest-version.sh") + if [[ ! -z "${version}" || "${version}" != "null" ]]; then + echo "${version}" | tee "./containers/apps/${app}/VERSION" > /dev/null + echo "App: ${app} using version: ${version}" + fi + fi + if test -f "./containers/apps/${app}/BASE"; then + if test -f "./containers/apps/${app}/latest-base.sh"; then + base=$(bash "./containers/apps/${app}/latest-base.sh") + if [[ ! -z "${base}" || "${base}" != "null" ]]; then + echo "${base}" | tee "containers/apps/${app}/BASE" > /dev/null + echo "App: ${app} using Base: ${base}" + fi + fi + fi + done + - name: Containers: Fetch new base versions + run: | + find ./containers/base -maxdepth 1 -mindepth 1 -type d -exec basename {} \; | while read app; do + if test -f "./containers/base/${app}/latest-version.sh"; then + version=$(bash "./containers/base/${app}/latest-version.sh") + if [[ ! -z "${version}" || "${version}" != "null" ]]; then + echo "${version}" | tee "./containers/base/${app}/VERSION" > /dev/null + echo "${app} ${version}" + fi + fi + done + + - name: (re)generate docs + run: | + # Templates to copy into each chart directory + readme_template="./tools/templates/docs/README.md.gotmpl" + config_template="./tools/templates/docs/CONFIG.md.gotmpl" + + for train in stable incubator develop non-free deprecated; do + for chart in charts/${train}/*; do + if [ -d "${chart}" ]; then + maxchartversion=$(cat ${chart}/Chart.yaml | grep "^version: " | awk -F" " '{ print $2 }') + chartname=$(basename ${chart}) + echo "-] Copying templates to ./${chart}" + helm-docs \ + --ignore-file=".helmdocsignore" \ + --output-file="README.md" \ + --template-files="./tools/templates/docs/README.md.gotmpl" \ + --chart-search-root="${chart}" + helm-docs \ + --ignore-file=".helmdocsignore" \ + --output-file="CONFIG.md" \ + --template-files="./tools/templates/docs/CONFIG.md.gotmpl" \ + --chart-search-root="${chart}" + fi + done + done + + - uses: dorny/paths-filter@v2 + id: filter + with: + list-files: json + base: HEAD + filters: | + changed: + - 'charts/stable/**' + - 'charts/incubator/**' + - 'charts/develop/**' + - 'charts/deprecated/**' + - 'charts/non-free/**' + - 'charts/library/**' + + - name: Filter filter-output + run: echo '${{ toJson(steps.filter.outputs) }}' > changes.json + + - name: Bump + run: | + APPS=$(jq --raw-output '.changed_files | fromjson | .[] |= sub("(?(?(?[\/]?)[^\/]+\/)(?(?[\/]?)[^\/]+\/)(?(?[\/]?)[^\/]+)(?.+))"; "\(.third_directory)") | unique' changes.json | jq -r '.[]') + echo "changed apps: ${APPS[*]}" + for chart in ${APPS[*]} + do + if [[ "${chart}" == '.gitkee' ]]; then + echo "Skipping..." + return + elif test -f "./charts/stable/${chart}/Chart.yaml"; then + train="stable" + elif test -f "./charts/incubator/${chart}/Chart.yaml"; then + train="incubator" + elif test -f "./charts/deprecated/${chart}/Chart.yaml"; then + train="deprecated" + elif test -f "./charts/non-free/${chart}/Chart.yaml"; then + train="non-free" + elif test -f "./charts/library/${chart}/Chart.yaml"; then + train="library" + else + train="develop" + fi + echo "Comparing versions for ${train}/${chart}" + echo "Bumping patch version for ${train}/${chart}" + pybump bump --file ./charts/${train}/${chart}/Chart.yaml --level patch + done + + - name: Copy general readme to website + run: | + yes | cp -rf index.yaml docs/index.yaml || echo "chart-index copy failed, continuing..." + yes | cp -rf .github/README.md docs/index.md || echo "readme copy failed, continuing..." + sed -i '1s/^/---\nhide:\n - navigation\n - toc\n---\n/' docs/index.md + sed -i 's~~~g' docs/index.md + yes | cp -rf .github/CODE_OF_CONDUCT docs/about/code_of_conduct.md || echo "CODE_OF_CONDUCT copy failed, continuing..." + yes | cp -rf .github/CONTRIBUTING docs/development/contributing.md || echo "CONTRIBUTING copy failed, continuing..." + yes | cp -rf .github/SUPPORT.md docs/manual/SUPPORT.md || echo "support policy copy failed, continuing..." + yes | cp -rf LICENSE docs/about/legal/LICENSE.md || echo "license copy failed, continuing..." + sed -i '1s/^/# License
\n\n/' docs/about/legal/LICENSE.md + yes | cp -rf NOTICE docs/about/legal/NOTICE.md || echo "license copy failed, continuing..." + sed -i '1s/^/# NOTICE
\n\n/' docs/about/legal/NOTICE.md + ls docs/ + + - name: Copy Apps readme to website + run: | + for train in stable incubator develop non-free deprecated; do + for chart in ${train}/*; do + if [ -d "${chart}" ]; then + chartname=$(basename ${chart}) + echo "Processing: ${chart}" + mkdir -p docs/apps/${train}/${chartname} || echo "app path already exists, continuing..." + yes | cp -rf ${chart}/README.md docs/apps/${train}/${chartname}/index.md || echo "readme copy failed, continuing..." + yes | cp -rf ${chart}/CONFIG.md docs/apps/${train}/${chartname}/CONFIG.md || echo "config copy failed, continuing..." + yes | cp -rf ${chart}/LICENSE docs/apps/${train}/${chartname}/LICENSE.md || echo "license copy failed, continuing..." + sed -i '1s/^/# License
\n\n/' docs/apps/${train}/${chartname}/LICENSE.md || echo "license edit failed, continuing..." + fi + done + echo "Processed charts for ${train}:" + ls docs/apps/${train}/ || echo "no Apps in train" + done + + - name: Cleanup + run: | + rm -rf changes.json + + - name: Commit and Push Housekeeping + run: | + git config user.name "TrueCharts-Bot" + git config user.email "bot@truecharts.org" + git add --all + git commit -sm "Daily Housekeeping" || exit 0 + git push diff --git a/tools/gen-helm-docs.sh b/tools/gen-helm-docs.sh deleted file mode 100755 index 0f9cdf4bd02..00000000000 --- a/tools/gen-helm-docs.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash -set -eu - -# Generate helm-docs for Helm charts -# Usage ./gen-helm-docs.sh [chart] - -# require helm-docs -command -v helm-docs >/dev/null 2>&1 || { - echo >&2 "helm-docs is not installed. Aborting." - exit 1 -} - -# Absolute path of repository -repository=$(git rev-parse --show-toplevel) - -# Templates to copy into each chart directory -readme_template="${repository}/tools/templates/docs/README.md.gotmpl" -config_template="${repository}/tools/templates/docs/CONFIG.md.gotmpl" -root="${repository}" - -for train in stable incubator develop non-free deprecated; do - for chart in charts/${train}/*; do - if [ -d "${chart}" ]; then - maxchartversion=$(cat ${chart}/Chart.yaml | grep "^version: " | awk -F" " '{ print $2 }') - chartname=$(basename ${chart}) - echo "-] Copying templates to ${repository}/${chart}" - helm-docs \ - --ignore-file=".helmdocsignore" \ - --output-file="README.md" \ - --template-files="${repository}/tools/templates/docs/README.md.gotmpl" \ - --chart-search-root="${chart}" - helm-docs \ - --ignore-file=".helmdocsignore" \ - --output-file="CONFIG.md" \ - --template-files="${repository}/tools/templates/docs/CONFIG.md.gotmpl" \ - --chart-search-root="${chart}" - fi - done -done