tests: re-add firefox e2e tests

Signed-off-by: Jens Langhammer <jens.langhammer@beryju.org>

azure-pipelines.yml

@@ -276,7 +276,7 @@ stages:
               targetPath: 'output-integration/'
               artifact: 'coverage-integration'
               publishLocation: 'pipeline'
-      - job: coverage_e2e
+      - job: coverage_e2e_chrome
         pool:
           vmImage: 'ubuntu-latest'
         steps:
@@ -318,6 +318,7 @@ stages:
             displayName: Run full test suite
             inputs:
               script: |
+                export BROWSER=chrome
                 pipenv run python -m scripts.generate_ci_config
                 pipenv run make test-e2e
           - task: CmdLine@2
@@ -331,20 +332,91 @@ stages:
             displayName: Prepare unittests and coverage for upload
             inputs:
               script: |
-                mkdir output-e2e
+                mkdir output-e2e-chrome
-                mv unittest.xml output-e2e/unittest.xml
+                mv unittest.xml output-e2e-chrome/unittest.xml
-                mv .coverage output-e2e/coverage
+                mv .coverage output-e2e-chrome/coverage
           - task: PublishPipelineArtifact@1
             condition: failed()
             displayName: Upload screenshots if selenium tests fail
             inputs:
               targetPath: 'selenium_screenshots/'
-              artifact: 'selenium screenshots'
+              artifact: 'chrome selenium screenshots'
               publishLocation: 'pipeline'
           - task: PublishPipelineArtifact@1
             inputs:
-              targetPath: 'output-e2e/'
+              targetPath: 'output-e2e-chrome/'
-              artifact: 'coverage-e2e'
+              artifact: 'coverage-e2e-chrome'
+              publishLocation: 'pipeline'
+      - job: coverage_e2e_firefox
+        pool:
+          vmImage: 'ubuntu-latest'
+        steps:
+          - task: UsePythonVersion@0
+            inputs:
+              versionSpec: '3.9'
+          - task: NodeTool@0
+            inputs:
+              versionSpec: '16.x'
+          - task: DockerCompose@0
+            displayName: Run services
+            inputs:
+              dockerComposeFile: 'scripts/ci.docker-compose.yml'
+              action: 'Run services'
+              buildImages: false
+          - task: CmdLine@2
+            inputs:
+              script: |
+                sudo apt update
+                sudo apt install -y libxmlsec1-dev pkg-config
+                sudo pip install -U wheel pipenv
+                pipenv install --dev --python python3.9
+          - task: DockerCompose@0
+            displayName: Run FirefoxDriver
+            inputs:
+              dockerComposeFile: 'tests/e2e/ci.docker-compose.yml'
+              action: 'Run a specific service'
+              serviceName: 'firefox'
+          - task: CmdLine@2
+            displayName: Build static files for e2e
+            inputs:
+              script: |
+                make gen-web
+                cd web
+                cd api && npm i && cd ..
+                npm i
+                npm run build
+          - task: CmdLine@2
+            displayName: Run full test suite
+            inputs:
+              script: |
+                export BROWSER=firefox
+                pipenv run python -m scripts.generate_ci_config
+                pipenv run make test-e2e
+          - task: CmdLine@2
+            condition: always()
+            displayName: Cleanup
+            inputs:
+              script: |
+                docker stop $(docker ps -aq)
+                docker container prune -f
+          - task: CmdLine@2
+            displayName: Prepare unittests and coverage for upload
+            inputs:
+              script: |
+                mkdir output-e2e-firefox
+                mv unittest.xml output-e2e-firefox/unittest.xml
+                mv .coverage output-e2e-firefox/coverage
+          - task: PublishPipelineArtifact@1
+            condition: failed()
+            displayName: Upload screenshots if selenium tests fail
+            inputs:
+              targetPath: 'selenium_screenshots/'
+              artifact: 'firefox selenium screenshots'
+              publishLocation: 'pipeline'
+          - task: PublishPipelineArtifact@1
+            inputs:
+              targetPath: 'output-e2e-firefox/'
+              artifact: 'coverage-e2e-firefox'
               publishLocation: 'pipeline'
   - stage: test_combine
     jobs:
@@ -355,8 +427,13 @@ stages:
           - task: DownloadPipelineArtifact@2
             inputs:
               buildType: 'current'
-              artifactName: 'coverage-e2e'
+              artifactName: 'coverage-e2e-firefox'
-              path: "coverage-e2e/"
+              path: "coverage-e2e-firefox/"
+          - task: DownloadPipelineArtifact@2
+            inputs:
+              buildType: 'current'
+              artifactName: 'coverage-e2e-chrome'
+              path: "coverage-e2e-chrome/"
           - task: DownloadPipelineArtifact@2
             inputs:
               buildType: 'current'
@@ -377,7 +454,7 @@ stages:
                 sudo apt install -y libxmlsec1-dev pkg-config
                 sudo pip install -U wheel pipenv
                 pipenv install --dev
-                pipenv run coverage combine coverage-e2e/coverage coverage-unittest/coverage coverage-integration/coverage
+                pipenv run coverage combine coverage-e2e-chrome/coverage coverage-e2e-firefox/coverage coverage-unittest/coverage coverage-integration/coverage
                 pipenv run coverage xml
                 pipenv run coverage html
           - task: PublishCodeCoverageResults@1
@@ -390,7 +467,8 @@ stages:
             inputs:
               testResultsFormat: 'JUnit'
               testResultsFiles: |
-                coverage-e2e/unittest.xml
+                coverage-e2e-chrome/unittest.xml
+                coverage-e2e-firefox/unittest.xml
                 coverage-integration/unittest.xml
                 coverage-unittest/unittest.xml
               mergeTestResults: true
@@ -403,7 +481,8 @@ stages:
                 npm install -g @zeus-ci/cli
                 npx zeus job update -b $BUILD_BUILDID -j $BUILD_BUILDNUMBER -r $BUILD_SOURCEVERSION
                 npx zeus upload -b $BUILD_BUILDID -j $BUILD_BUILDNUMBER -t "application/x-cobertura+xml" coverage.xml
-                npx zeus upload -b $BUILD_BUILDID -j $BUILD_BUILDNUMBER -t "application/x-junit+xml" coverage-e2e/unittest.xml
+                npx zeus upload -b $BUILD_BUILDID -j $BUILD_BUILDNUMBER -t "application/x-junit+xml" coverage-e2e-firefox/unittest.xml
+                npx zeus upload -b $BUILD_BUILDID -j $BUILD_BUILDNUMBER -t "application/x-junit+xml" coverage-e2e-chrome/unittest.xml
                 npx zeus upload -b $BUILD_BUILDID -j $BUILD_BUILDNUMBER -t "application/x-junit+xml" coverage-integration/unittest.xml
                 npx zeus upload -b $BUILD_BUILDID -j $BUILD_BUILDNUMBER -t "application/x-junit+xml" coverage-unittest/unittest.xml
                 npx zeus job update --status=passed -b $BUILD_BUILDID -j $BUILD_BUILDNUMBER -r $BUILD_SOURCEVERSION

tests/e2e/ci.docker-compose.yml

@@ -6,3 +6,8 @@ services:
     volumes:
       - /dev/shm:/dev/shm
     network_mode: host
+  firefox:
+    image: selenium/standalone-firefox:3.141.59-20210713
+    volumes:
+      - /dev/shm:/dev/shm
+    network_mode: host

tests/e2e/utils.py

@@ -21,7 +21,6 @@ from selenium.common.exceptions import (
     WebDriverException,
 )
 from selenium.webdriver.common.by import By
-from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
 from selenium.webdriver.common.keys import Keys
 from selenium.webdriver.remote.webdriver import WebDriver
 from selenium.webdriver.remote.webelement import WebElement
@@ -33,6 +32,8 @@ from authentik.core.models import User
 from authentik.managed.manager import ObjectManager
 
 RETRIES = int(environ.get("RETRIES", "5"))
+BROWSER_NAME = environ.get("BROWSER", "chrome")
+
 
 # pylint: disable=invalid-name
 def USER() -> User:  # noqa
@@ -88,7 +89,11 @@ class SeleniumTestCase(StaticLiveServerTestCase):
     def _get_driver(self) -> WebDriver:
         return webdriver.Remote(
             command_executor="http://localhost:4444/wd/hub",
-            desired_capabilities=DesiredCapabilities.CHROME,
+            desired_capabilities={
+                "browserName": BROWSER_NAME,
+                "version": "",
+                "platform": "ANY",
+            }
         )
 
     def tearDown(self):