name: Build and Release

run-name: ${{ github.event.inputs.publish == 'true' && '[Prod]' || '[Dev]' }} Build ${{ github.run_number }} of branch ${{ github.ref_name }} by @${{ github.actor }}

on:

push:

tags:

- '*'

workflow_dispatch:

inputs:

summary:

description: 'Release Summary'

required: false

type: string

default: ''

publish:

description: 'Create Production Release'

default: false

required: true

type: boolean

sandbox:

description: 'Deploy to Sandbox'

default: true

required: true

type: boolean

skiptests:

description: 'Skip Tests'

default: false

required: true

type: boolean

ignoreLowerCoverage:

description: 'Ignore Lower Coverage'

default: false

required: true

type: boolean

updateCoverage:

description: 'Update Baseline Coverage'

default: false

required: true

type: boolean

dryrun:

description: 'Dry Run'

default: false

required: true

type: boolean

jobs:

# -----------------------------------------------------------------

# PREPARE

# -----------------------------------------------------------------

prepare:

name: Prepare Release

runs-on: ubuntu-latest

outputs:

should_deploy: ${{ steps.buildvars.outputs.should_deploy }}

pkg_version: ${{ steps.buildvars.outputs.pkg_version }}

from_tag: ${{ steps.semver.outputs.nextStrict }}

to_tag: ${{ steps.semver.outputs.current }}

steps:

- uses: actions/checkout@v3

with:

fetch-depth: 0

- name: Dry Run Notify

if: ${{ github.event.inputs.dryrun == 'true' }}

run: |

echo "::notice::This is a DRY RUN of a production release. No release will be created."

- name: Get Next Version

if: ${{ github.event.inputs.publish == 'true' || github.event.inputs.dryrun == 'true' }}

id: semver

uses: ietf-tools/semver-action@v1

with:

token: ${{ github.token }}

branch: main

skipInvalidTags: true

- name: Set Next Version Env Var

if: ${{ github.event.inputs.publish == 'true' || github.event.inputs.dryrun == 'true' }}

run: |

echo "NEXT_VERSION=$nextStrict" >> $GITHUB_ENV

- name: Create Draft Release

uses: ncipollo/release-action@v1.12.0

if: ${{ github.event.inputs.publish == 'true' && github.event.inputs.dryrun == 'false' }}

with:

prerelease: true

draft: false

commit: ${{ github.sha }}

tag: ${{ env.NEXT_VERSION }}

name: ${{ env.NEXT_VERSION }}

body: '*pending*'

token: ${{ secrets.GITHUB_TOKEN }}

- name: Set Build Variables

id: buildvars

run: |

if [[ $NEXT_VERSION ]]; then

echo "Using AUTO SEMVER mode: $NEXT_VERSION"

echo "should_deploy=true" >> $GITHUB_OUTPUT

echo "pkg_version=$NEXT_VERSION" >> $GITHUB_OUTPUT

echo "::notice::Release $NEXT_VERSION created using branch $GITHUB_REF_NAME"

elif [[ "$GITHUB_REF" =~ ^refs/tags/* ]]; then

echo "Using TAG mode: $GITHUB_REF_NAME"

echo "should_deploy=true" >> $GITHUB_OUTPUT

echo "pkg_version=$GITHUB_REF_NAME" >> $GITHUB_OUTPUT

echo "::notice::Release $GITHUB_REF_NAME created using tag $GITHUB_REF_NAME"

else

echo "Using TEST mode: 9.0.0-dev.$GITHUB_RUN_NUMBER"

echo "should_deploy=false" >> $GITHUB_OUTPUT

echo "pkg_version=9.0.0-dev.$GITHUB_RUN_NUMBER" >> $GITHUB_OUTPUT

echo "::notice::Non-production build 9.0.0-dev.$GITHUB_RUN_NUMBER created using branch $GITHUB_REF_NAME"

fi

# -----------------------------------------------------------------

# TESTS

# -----------------------------------------------------------------

tests-python:

name: Run Tests (Python)

if: ${{ github.event.inputs.skiptests == 'false' }}

needs: [prepare]

runs-on: ubuntu-latest

container: ghcr.io/ietf-tools/datatracker-app-base:latest

services:

db:

image: ghcr.io/ietf-tools/datatracker-db:latest

volumes:

- mariadb-data:/var/lib/mysql

env:

MYSQL_ROOT_PASSWORD: ietf

MYSQL_DATABASE: ietf_utf8

MYSQL_USER: django

MYSQL_PASSWORD: RkTkDPFnKpko

steps:

- uses: actions/checkout@v3

- name: Prepare for tests

run: |

chmod +x ./dev/tests/prepare.sh

sh ./dev/tests/prepare.sh

- name: Ensure DB is ready

run: |

/usr/local/bin/wait-for db:3306 -- echo "DB ready"

- name: Run all tests

shell: bash

run: |

echo "Running checks..."

./ietf/manage.py check

./ietf/manage.py migrate --fake-initial

echo "Validating migrations..."

if ! ( ietf/manage.py makemigrations --dry-run --check --verbosity 3 ) ; then

echo "Model changes without migrations found."

exit 1

fi

echo "Running tests..."

if [[ "x${{ github.event.inputs.ignoreLowerCoverage }}" == "xtrue" ]]; then

echo "Lower coverage failures will be ignored."

./ietf/manage.py test --settings=settings_sqlitetest --ignore-lower-coverage

else

./ietf/manage.py test --settings=settings_sqlitetest

fi

coverage xml

- name: Upload Coverage Results to Codecov

uses: codecov/codecov-action@v3.1.1

with:

files: coverage.xml

- name: Convert Coverage Results

if: ${{ always() }}

run: |

mv latest-coverage.json coverage.json

- name: Upload Coverage Results as Build Artifact

uses: actions/upload-artifact@v3

if: ${{ always() }}

with:

name: coverage

path: coverage.json

tests-playwright:

name: Run Tests (Playwright)

if: ${{ github.event.inputs.skiptests == 'false' }}

needs: [prepare]

runs-on: macos-latest

strategy:

fail-fast: false

matrix:

project: [chromium, firefox]

steps:

- uses: actions/checkout@v3

- uses: actions/setup-node@v3

with:

node-version: '18'

- name: Run all tests

run: |

echo "Installing dependencies..."

yarn

echo "Installing Playwright..."

cd playwright

mkdir test-results

npm ci

npx playwright install --with-deps ${{ matrix.project }}

echo "Running tests..."

npx playwright test --project=${{ matrix.project }}

- name: Upload Report

uses: actions/upload-artifact@v3

if: ${{ always() }}

continue-on-error: true

with:

name: playwright-results-${{ matrix.project }}

path: playwright/test-results/

if-no-files-found: ignore

tests-playwright-legacy:

name: Run Tests (Playwright Legacy)

if: ${{ github.event.inputs.skiptests == 'false' }}

needs: [prepare]

runs-on: ubuntu-latest

container: ghcr.io/ietf-tools/datatracker-app-base:latest

strategy:

fail-fast: false

matrix:

project: [chromium, firefox]

services:

db:

image: ghcr.io/ietf-tools/datatracker-db:latest

volumes:

- mariadb-data:/var/lib/mysql

env:

MYSQL_ROOT_PASSWORD: ietf

MYSQL_DATABASE: ietf_utf8

MYSQL_USER: django

MYSQL_PASSWORD: RkTkDPFnKpko

steps:

- uses: actions/checkout@v3

- name: Prepare for tests

run: |

chmod +x ./dev/tests/prepare.sh

sh ./dev/tests/prepare.sh

- name: Ensure DB is ready

run: |

/usr/local/bin/wait-for db:3306 -- echo "DB ready"

- name: Start Datatracker

run: |

echo "Running checks..."

./ietf/manage.py check

echo "Starting datatracker..."

./ietf/manage.py runserver 0.0.0.0:8000 --settings=settings_local &

echo "Waiting for datatracker to be ready..."

/usr/local/bin/wait-for localhost:8000 -- echo "Datatracker ready"

- name: Run all tests

env:

# Required to get firefox to run as root:

HOME: ""

run: |

echo "Installing dependencies..."

yarn

echo "Installing Playwright..."

cd playwright

mkdir test-results

npm ci

npx playwright install --with-deps ${{ matrix.project }}

echo "Running tests..."

npx playwright test --project=${{ matrix.project }} -c playwright-legacy.config.js

- name: Upload Report

uses: actions/upload-artifact@v3

if: ${{ always() }}

continue-on-error: true

with:

name: playwright-legacy-results-${{ matrix.project }}

path: playwright/test-results/

if-no-files-found: ignore

# -----------------------------------------------------------------

# RELEASE

# -----------------------------------------------------------------

release:

name: Make Release

if: ${{ always() }}

needs: [tests-python, tests-playwright, tests-playwright-legacy, prepare]

runs-on: ubuntu-latest

env:

SHOULD_DEPLOY: ${{needs.prepare.outputs.should_deploy}}

PKG_VERSION: ${{needs.prepare.outputs.pkg_version}}

FROM_TAG: ${{needs.prepare.outputs.from_tag}}

TO_TAG: ${{needs.prepare.outputs.to_tag}}

steps:

- uses: actions/checkout@v3

with:

fetch-depth: 0

- name: Setup Node.js

uses: actions/setup-node@v3

with:

node-version: '16'

- name: Setup Python

uses: actions/setup-python@v4

with:

python-version: '3.x'

- name: Download a Coverage Results

if: ${{ github.event.inputs.skiptests == 'false' }}

uses: actions/download-artifact@v3.0.2

with:

name: coverage

- name: Make Release Build

env:

DEBIAN_FRONTEND: noninteractive

run: |

echo "PKG_VERSION: $PKG_VERSION"

echo "GITHUB_SHA: $GITHUB_SHA"

echo "GITHUB_REF_NAME: $GITHUB_REF_NAME"

echo "Running build script..."

chmod +x ./dev/deploy/build.sh

sh ./dev/deploy/build.sh

echo "Setting version $PKG_VERSION..."

sed -i -r -e "s|^__version__ += '.*'$|__version__ = '$PKG_VERSION'|" ietf/__init__.py

sed -i -r -e "s|^__release_hash__ += '.*'$|__release_hash__ = '$GITHUB_SHA'|" ietf/__init__.py

sed -i -r -e "s|^__release_branch__ += '.*'$|__release_branch__ = '$GITHUB_REF_NAME'|" ietf/__init__.py

- name: Set Production Flags

if: ${{ env.SHOULD_DEPLOY == 'true' }}

run: |

echo "Setting production flags in settings.py..."

sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = False/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'production'/" ietf/settings.py

- name: Make Release Tarball

env:

DEBIAN_FRONTEND: noninteractive

run: |

echo "Build release tarball..."

mkdir -p /home/runner/work/release

tar -czf /home/runner/work/release/release.tar.gz -X dev/deploy/exclude-patterns.txt .

- name: Update CHANGELOG

id: changelog

uses: Requarks/changelog-action@v1

if: ${{ env.SHOULD_DEPLOY == 'true' && github.event.inputs.dryrun == 'false' }}

with:

token: ${{ github.token }}

fromTag: ${{ env.FROM_TAG }}

toTag: ${{ env.TO_TAG }}

writeToFile: false

- name: Prepare Coverage Action

if: ${{ github.event.inputs.skiptests == 'false' }}

working-directory: ./dev/coverage-action

run: npm install

- name: Process Coverage Stats + Chart

id: covprocess

uses: ./dev/coverage-action/

if: ${{ github.event.inputs.skiptests == 'false' }}

with:

token: ${{ github.token }}

tokenCommon: ${{ secrets.GH_COMMON_TOKEN }}

repoCommon: common

version: ${{needs.prepare.outputs.pkg_version}}

changelog: ${{ steps.changelog.outputs.changes }}

summary: ${{ github.event.inputs.summary }}

coverageResultsPath: coverage.json

histCoveragePath: historical-coverage.json

- name: Create Release

uses: ncipollo/release-action@v1.12.0

if: ${{ env.SHOULD_DEPLOY == 'true' && github.event.inputs.dryrun == 'false' }}

with:

allowUpdates: true

makeLatest: true

draft: false

tag: ${{ env.PKG_VERSION }}

name: ${{ env.PKG_VERSION }}

body: ${{ steps.covprocess.outputs.changelog }}

artifacts: "/home/runner/work/release/release.tar.gz,coverage.json,historical-coverage.json"

token: ${{ secrets.GITHUB_TOKEN }}

- name: Update Baseline Coverage

uses: ncipollo/release-action@v1.12.0

if: ${{ github.event.inputs.updateCoverage == 'true' && github.event.inputs.dryrun == 'false' }}

with:

allowUpdates: true

tag: baseline

omitBodyDuringUpdate: true

omitNameDuringUpdate: true

omitPrereleaseDuringUpdate: true

replacesArtifacts: true

artifacts: "coverage.json"

token: ${{ secrets.GITHUB_TOKEN }}

- name: Upload Build Artifacts

uses: actions/upload-artifact@v3

with:

name: release-${{ env.PKG_VERSION }}

path: /home/runner/work/release/release.tar.gz

- name: Notify on Slack

if: ${{ always() }}

uses: slackapi/slack-github-action@v1.23.0

with:

channel-id: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }}

payload: |

{

"text": "Datatracker - Build <https://github.com/ietf-tools/datatracker/actions/runs/${{ github.run_id }}|${{ env.PKG_VERSION }}> by ${{ github.triggering_actor }} completed - <@${{ secrets.SLACK_UID_RJSPARKS }}>"

}

env:

SLACK_BOT_TOKEN: ${{ secrets.SLACK_GH_BOT }}

# -----------------------------------------------------------------

# SANDBOX

# -----------------------------------------------------------------

sandbox:

name: Deploy to Sandbox

if: ${{ always() && github.event.inputs.sandbox == 'true' }}

needs: [prepare, release]

runs-on: [self-hosted, dev-server]

env:

PKG_VERSION: ${{needs.prepare.outputs.pkg_version}}

steps:

- uses: actions/checkout@v3

- name: Download a Release Artifact

uses: actions/download-artifact@v3.0.2

with:

name: release-${{ env.PKG_VERSION }}

- name: Deploy to containers

env:

DEBIAN_FRONTEND: noninteractive

run: |

echo "Reset production flags in settings.py..."

sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = True/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'development'/" ietf/settings.py

echo "Install Deploy to Container CLI dependencies..."

cd dev/deploy-to-container

npm ci

cd ../..

echo "Start Deploy..."

node ./dev/deploy-to-container/cli.js --branch ${{ github.ref_name }} --domain dev.ietf.org --appversion ${{ env.PKG_VERSION }} --commit ${{ github.sha }} --ghrunid ${{ github.run_id }}

- name: Cleanup old docker resources

env:

DEBIAN_FRONTEND: noninteractive

run: |

docker image prune -a -f