ci: use native caching (#2665)

* ci: use native caching

* migrate to actions

* cleanup

* migrate more

.github/actions/docker-setup/action.yml

@@ -0,0 +1,49 @@
+name: 'Prepare docker environment variables'
+description: 'Prepare docker environment variables'
+
+outputs:
+  shouldBuild:
+    description: "Whether to build image or not"
+    value: ${{ steps.ev.outputs.shouldBuild }}
+  branchName:
+    description: "Branch name"
+    value: ${{ steps.ev.outputs.branchName }}
+  branchNameContainer:
+    description: "Branch name (for containers)"
+    value: ${{ steps.ev.outputs.branchNameContainer }}
+  timestamp:
+    description: "Timestamp"
+    value: ${{ steps.ev.outputs.timestamp }}
+  sha:
+    description: "sha"
+    value: ${{ steps.ev.outputs.sha }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Generate config
+      id: ev
+      shell: python
+      run: |
+        """Helper script to get the actual branch name, docker safe"""
+        import os
+        from time import time
+
+        env_pr_branch = "GITHUB_HEAD_REF"
+        default_branch = "GITHUB_REF"
+        sha = "GITHUB_SHA"
+
+        branch_name = os.environ[default_branch]
+        if os.environ.get(env_pr_branch, "") != "":
+            branch_name = os.environ[env_pr_branch]
+
+        should_build = str(os.environ.get("DOCKER_USERNAME", "") != "").lower()
+
+        print("##[set-output name=branchName]%s" % branch_name)
+        print(
+            "##[set-output name=branchNameContainer]%s"
+            % branch_name.replace("refs/heads/", "").replace("/", "-")
+        )
+        print("##[set-output name=timestamp]%s" % int(time()))
+        print("##[set-output name=sha]%s" % os.environ[sha])
+        print("##[set-output name=shouldBuild]%s" % should_build)

.github/actions/setup/action.yml

@@ -0,0 +1,45 @@
+name: 'Setup authentik testing environemnt'
+description: 'Setup authentik testing environemnt'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install poetry
+      shell: bash
+      run: |
+        pipx install poetry || true
+        sudo apt update
+        sudo apt install -y libxmlsec1-dev pkg-config gettext
+    - name: Setup python and restore poetry
+      uses: actions/setup-python@v3
+      with:
+        python-version: '3.10'
+        cache: 'poetry'
+    - name: Setup node
+      uses: actions/setup-node@v3.1.0
+      with:
+        node-version: '16'
+        cache: 'npm'
+        cache-dependency-path: web/package-lock.json
+    - name: Setup dependencies
+      shell: bash
+      run: |
+        docker-compose -f .github/actions/setup/docker-compose.yml up -d
+        poetry env use python3.10
+        poetry install
+        npm install -g pyright@1.1.136
+    - name: Generate config
+      shell: poetry run python {0}
+      run: |
+        from authentik.lib.generators import generate_id
+        from yaml import safe_dump
+
+        with open("local.env.yml", "w") as _config:
+            safe_dump(
+                {
+                    "log_level": "debug",
+                    "secret_key": generate_id(),
+                },
+                _config,
+                default_flow_style=False,
+            )

.github/workflows/ci-main.yml

@@ -32,35 +32,16 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-python@v3
+      - name: Setup authentik env
-      - uses: actions/setup-node@v3.1.0
+        uses: ./.github/actions/setup
-        with:
-          node-version: '16'
-      - id: cache-poetry
-        uses: actions/cache@v3
-        with:
-          path: ~/.cache/pypoetry/virtualenvs
-          key: ${{ runner.os }}-poetry-cache-v2-${{ hashFiles('**/poetry.lock') }}
-      - name: prepare
-        env:
-          INSTALL: ${{ steps.cache-poetry.outputs.cache-hit }}
-        run: scripts/ci_prepare.sh
       - name: run job
         run: poetry run make ci-${{ matrix.job }}
   test-migrations:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-python@v3
+      - name: Setup authentik env
-      - id: cache-poetry
+        uses: ./.github/actions/setup
-        uses: actions/cache@v3
-        with:
-          path: ~/.cache/pypoetry/virtualenvs
-          key: ${{ runner.os }}-poetry-cache-v2-${{ hashFiles('**/poetry.lock') }}
-      - name: prepare
-        env:
-          INSTALL: ${{ steps.cache-poetry.outputs.cache-hit }}
-        run: scripts/ci_prepare.sh
       - name: run migrations
         run: poetry run python -m lifecycle.migrate
   test-migrations-from-stable:
@@ -69,17 +50,8 @@ jobs:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      - uses: actions/setup-python@v3
+      - name: Setup authentik env
-      - name: prepare variables
+        uses: ./.github/actions/setup
-        id: ev
-        run: |
-          python ./scripts/gh_env.py
-          sudo pip install -U pipenv
-      - id: cache-poetry
-        uses: actions/cache@v3
-        with:
-          path: ~/.cache/pypoetry/virtualenvs
-          key: ${{ runner.os }}-poetry-cache-v2-${{ hashFiles('**/poetry.lock') }}
       - name: checkout stable
         run: |
           # Copy current, latest config to local
@@ -89,13 +61,8 @@ jobs:
           git checkout $(git describe --abbrev=0 --match 'version/*')
           rm -rf .github/ scripts/
           mv ../.github ../scripts .
-      - name: prepare
+      - name: Setup authentik env (ensure stable deps are installed)
-        env:
+        uses: ./.github/actions/setup
-          INSTALL: ${{ steps.cache-poetry.outputs.cache-hit }}
-        run: |
-          scripts/ci_prepare.sh
-          # install anyways since stable will have different dependencies
-          poetry install
       - name: run migrations to stable
         run: poetry run python -m lifecycle.migrate
       - name: checkout current code
@@ -103,28 +70,19 @@ jobs:
           set -x
           git fetch
           git reset --hard HEAD
+          git clean -d -fx .
           git checkout $GITHUB_SHA
           poetry install
-      - name: prepare
+      - name: Setup authentik env (ensure latest deps are installed)
-        env:
+        uses: ./.github/actions/setup
-          INSTALL: ${{ steps.cache-poetry.outputs.cache-hit }}
-        run: scripts/ci_prepare.sh
       - name: migrate to latest
         run: poetry run python -m lifecycle.migrate
   test-unittest:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-python@v3
+      - name: Setup authentik env
-      - id: cache-poetry
+        uses: ./.github/actions/setup
-        uses: actions/cache@v3
-        with:
-          path: ~/.cache/pypoetry/virtualenvs
-          key: ${{ runner.os }}-poetry-cache-v2-${{ hashFiles('**/poetry.lock') }}
-      - name: prepare
-        env:
-          INSTALL: ${{ steps.cache-poetry.outputs.cache-hit }}
-        run: scripts/ci_prepare.sh
       - uses: testspace-com/setup-testspace@v1
         with:
           domain: ${{github.repository_owner}}
@@ -142,16 +100,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-python@v3
+      - name: Setup authentik env
-      - id: cache-poetry
+        uses: ./.github/actions/setup
-        uses: actions/cache@v3
-        with:
-          path: ~/.cache/pypoetry/virtualenvs
-          key: ${{ runner.os }}-poetry-cache-v2-${{ hashFiles('**/poetry.lock') }}
-      - name: prepare
-        env:
-          INSTALL: ${{ steps.cache-poetry.outputs.cache-hit }}
-        run: scripts/ci_prepare.sh
       - uses: testspace-com/setup-testspace@v1
         with:
           domain: ${{github.repository_owner}}
@@ -171,25 +121,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-python@v3
+      - name: Setup authentik env
-      - uses: actions/setup-node@v3.1.0
+        uses: ./.github/actions/setup
-        with:
-          node-version: '16'
-          cache: 'npm'
-          cache-dependency-path: web/package-lock.json
       - uses: testspace-com/setup-testspace@v1
         with:
           domain: ${{github.repository_owner}}
-      - id: cache-poetry
+      - name: Setup authentik env
-        uses: actions/cache@v3
-        with:
-          path: ~/.cache/pypoetry/virtualenvs
-          key: ${{ runner.os }}-poetry-cache-v2-${{ hashFiles('**/poetry.lock') }}
-      - name: prepare
-        env:
-          INSTALL: ${{ steps.cache-poetry.outputs.cache-hit }}
         run: |
-          scripts/ci_prepare.sh
           docker-compose -f tests/e2e/docker-compose.yml up -d
       - id: cache-web
         uses: actions/cache@v3
@@ -216,25 +154,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-python@v3
+      - name: Setup authentik env
-      - uses: actions/setup-node@v3.1.0
+        uses: ./.github/actions/setup
-        with:
-          node-version: '16'
-          cache: 'npm'
-          cache-dependency-path: web/package-lock.json
       - uses: testspace-com/setup-testspace@v1
         with:
           domain: ${{github.repository_owner}}
-      - id: cache-poetry
+      - name: Setup authentik env
-        uses: actions/cache@v3
-        with:
-          path: ~/.cache/pypoetry/virtualenvs
-          key: ${{ runner.os }}-poetry-cache-v2-${{ hashFiles('**/poetry.lock') }}
-      - name: prepare
-        env:
-          INSTALL: ${{ steps.cache-poetry.outputs.cache-hit }}
         run: |
-          scripts/ci_prepare.sh
           docker-compose -f tests/e2e/docker-compose.yml up -d
       - id: cache-web
         uses: actions/cache@v3
@@ -288,8 +214,7 @@ jobs:
         id: ev
         env:
           DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
-        run: |
+        uses: ./.github/actions/docker-setup
-          python ./scripts/gh_env.py
       - name: Login to Container Registry
         uses: docker/login-action@v1
         if: ${{ steps.ev.outputs.shouldBuild == 'true' }}

.github/workflows/ci-outpost.yml

@@ -68,10 +68,9 @@ jobs:
         uses: docker/setup-buildx-action@v1
       - name: prepare variables
         id: ev
+        uses: ./.github/actions/docker-setup
         env:
           DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
-        run: |
-          python ./scripts/gh_env.py
       - name: Login to Container Registry
         uses: docker/login-action@v1
         if: ${{ steps.ev.outputs.shouldBuild == 'true' }}

.github/workflows/translation-compile.yml

@@ -21,19 +21,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-python@v3
+      - name: Setup authentik env
-      - id: cache-poetry
+        uses: ./.github/actions/setup
-        uses: actions/cache@v3
-        with:
-          path: ~/.cache/pypoetry/virtualenvs
-          key: ${{ runner.os }}-poetry-cache-v2-${{ hashFiles('**/poetry.lock') }}
-      - name: prepare
-        env:
-          INSTALL: ${{ steps.cache-poetry.outputs.cache-hit }}
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y gettext
-          scripts/ci_prepare.sh
       - name: run compile
         run: poetry run ./manage.py compilemessages
       - name: Create Pull Request

scripts/ci_prepare.sh

@@ -1,12 +0,0 @@
-#!/bin/bash -xe
-docker-compose -f scripts/ci.docker-compose.yml up -d
-
-sudo apt update
-sudo apt install -y libxmlsec1-dev pkg-config
-python3 -m pip install -U wheel poetry
-poetry env use python3.10
-if [[ "$INSTALL" != "true" ]]; then
-    poetry install
-fi
-poetry run python -m scripts.generate_ci_config
-npm install -g pyright@1.1.136

scripts/generate_ci_config.py

@@ -1,13 +0,0 @@
-"""Utility script to generate a config for CI runs"""
-from authentik.lib.generators import generate_id
-from yaml import safe_dump
-
-with open("local.env.yml", "w") as _config:
-    safe_dump(
-        {
-            "log_level": "debug",
-            "secret_key": generate_id(),
-        },
-        _config,
-        default_flow_style=False,
-    )

scripts/gh_env.py

@@ -1,22 +0,0 @@
-"""Helper script to get the actual branch name, docker safe"""
-import os
-from time import time
-
-env_pr_branch = "GITHUB_HEAD_REF"
-default_branch = "GITHUB_REF"
-sha = "GITHUB_SHA"
-
-branch_name = os.environ[default_branch]
-if os.environ.get(env_pr_branch, "") != "":
-    branch_name = os.environ[env_pr_branch]
-
-should_build = str(os.environ.get("DOCKER_USERNAME", "") != "").lower()
-
-print("##[set-output name=branchName]%s" % branch_name)
-print(
-    "##[set-output name=branchNameContainer]%s"
-    % branch_name.replace("refs/heads/", "").replace("/", "-")
-)
-print("##[set-output name=timestamp]%s" % int(time()))
-print("##[set-output name=sha]%s" % os.environ[sha])
-print("##[set-output name=shouldBuild]%s" % should_build)