diff --git a/azure-pipelines.yml b/azure-pipelines.yml index aeb8d7305..0d82f673d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -276,7 +276,7 @@ stages: targetPath: 'output-integration/' artifact: 'coverage-integration' publishLocation: 'pipeline' - - job: coverage_e2e + - job: coverage_e2e_chrome pool: vmImage: 'ubuntu-latest' steps: @@ -318,6 +318,7 @@ stages: displayName: Run full test suite inputs: script: | + export BROWSER=chrome pipenv run python -m scripts.generate_ci_config pipenv run make test-e2e - task: CmdLine@2 @@ -331,20 +332,91 @@ stages: displayName: Prepare unittests and coverage for upload inputs: script: | - mkdir output-e2e - mv unittest.xml output-e2e/unittest.xml - mv .coverage output-e2e/coverage + mkdir output-e2e-chrome + mv unittest.xml output-e2e-chrome/unittest.xml + mv .coverage output-e2e-chrome/coverage - task: PublishPipelineArtifact@1 condition: failed() displayName: Upload screenshots if selenium tests fail inputs: targetPath: 'selenium_screenshots/' - artifact: 'selenium screenshots' + artifact: 'chrome selenium screenshots' publishLocation: 'pipeline' - task: PublishPipelineArtifact@1 inputs: - targetPath: 'output-e2e/' - artifact: 'coverage-e2e' + targetPath: 'output-e2e-chrome/' + artifact: 'coverage-e2e-chrome' + publishLocation: 'pipeline' + - job: coverage_e2e_firefox + pool: + vmImage: 'ubuntu-latest' + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.9' + - task: NodeTool@0 + inputs: + versionSpec: '16.x' + - task: DockerCompose@0 + displayName: Run services + inputs: + dockerComposeFile: 'scripts/ci.docker-compose.yml' + action: 'Run services' + buildImages: false + - task: CmdLine@2 + inputs: + script: | + sudo apt update + sudo apt install -y libxmlsec1-dev pkg-config + sudo pip install -U wheel pipenv + pipenv install --dev --python python3.9 + - task: DockerCompose@0 + displayName: Run FirefoxDriver + inputs: + dockerComposeFile: 'tests/e2e/ci.docker-compose.yml' + action: 'Run a specific service' + serviceName: 'firefox' + - task: CmdLine@2 + displayName: Build static files for e2e + inputs: + script: | + make gen-web + cd web + cd api && npm i && cd .. + npm i + npm run build + - task: CmdLine@2 + displayName: Run full test suite + inputs: + script: | + export BROWSER=firefox + pipenv run python -m scripts.generate_ci_config + pipenv run make test-e2e + - task: CmdLine@2 + condition: always() + displayName: Cleanup + inputs: + script: | + docker stop $(docker ps -aq) + docker container prune -f + - task: CmdLine@2 + displayName: Prepare unittests and coverage for upload + inputs: + script: | + mkdir output-e2e-firefox + mv unittest.xml output-e2e-firefox/unittest.xml + mv .coverage output-e2e-firefox/coverage + - task: PublishPipelineArtifact@1 + condition: failed() + displayName: Upload screenshots if selenium tests fail + inputs: + targetPath: 'selenium_screenshots/' + artifact: 'firefox selenium screenshots' + publishLocation: 'pipeline' + - task: PublishPipelineArtifact@1 + inputs: + targetPath: 'output-e2e-firefox/' + artifact: 'coverage-e2e-firefox' publishLocation: 'pipeline' - stage: test_combine jobs: @@ -355,8 +427,13 @@ stages: - task: DownloadPipelineArtifact@2 inputs: buildType: 'current' - artifactName: 'coverage-e2e' - path: "coverage-e2e/" + artifactName: 'coverage-e2e-firefox' + path: "coverage-e2e-firefox/" + - task: DownloadPipelineArtifact@2 + inputs: + buildType: 'current' + artifactName: 'coverage-e2e-chrome' + path: "coverage-e2e-chrome/" - task: DownloadPipelineArtifact@2 inputs: buildType: 'current' @@ -377,7 +454,7 @@ stages: sudo apt install -y libxmlsec1-dev pkg-config sudo pip install -U wheel pipenv pipenv install --dev - pipenv run coverage combine coverage-e2e/coverage coverage-unittest/coverage coverage-integration/coverage + pipenv run coverage combine coverage-e2e-chrome/coverage coverage-e2e-firefox/coverage coverage-unittest/coverage coverage-integration/coverage pipenv run coverage xml pipenv run coverage html - task: PublishCodeCoverageResults@1 @@ -390,7 +467,8 @@ stages: inputs: testResultsFormat: 'JUnit' testResultsFiles: | - coverage-e2e/unittest.xml + coverage-e2e-chrome/unittest.xml + coverage-e2e-firefox/unittest.xml coverage-integration/unittest.xml coverage-unittest/unittest.xml mergeTestResults: true @@ -403,7 +481,8 @@ stages: npm install -g @zeus-ci/cli npx zeus job update -b $BUILD_BUILDID -j $BUILD_BUILDNUMBER -r $BUILD_SOURCEVERSION npx zeus upload -b $BUILD_BUILDID -j $BUILD_BUILDNUMBER -t "application/x-cobertura+xml" coverage.xml - npx zeus upload -b $BUILD_BUILDID -j $BUILD_BUILDNUMBER -t "application/x-junit+xml" coverage-e2e/unittest.xml + npx zeus upload -b $BUILD_BUILDID -j $BUILD_BUILDNUMBER -t "application/x-junit+xml" coverage-e2e-firefox/unittest.xml + npx zeus upload -b $BUILD_BUILDID -j $BUILD_BUILDNUMBER -t "application/x-junit+xml" coverage-e2e-chrome/unittest.xml npx zeus upload -b $BUILD_BUILDID -j $BUILD_BUILDNUMBER -t "application/x-junit+xml" coverage-integration/unittest.xml npx zeus upload -b $BUILD_BUILDID -j $BUILD_BUILDNUMBER -t "application/x-junit+xml" coverage-unittest/unittest.xml npx zeus job update --status=passed -b $BUILD_BUILDID -j $BUILD_BUILDNUMBER -r $BUILD_SOURCEVERSION diff --git a/tests/e2e/ci.docker-compose.yml b/tests/e2e/ci.docker-compose.yml index 5be532db3..d6295cabb 100644 --- a/tests/e2e/ci.docker-compose.yml +++ b/tests/e2e/ci.docker-compose.yml @@ -6,3 +6,8 @@ services: volumes: - /dev/shm:/dev/shm network_mode: host + firefox: + image: selenium/standalone-firefox:3.141.59-20210713 + volumes: + - /dev/shm:/dev/shm + network_mode: host diff --git a/tests/e2e/utils.py b/tests/e2e/utils.py index 5ace1eab0..d357f6c24 100644 --- a/tests/e2e/utils.py +++ b/tests/e2e/utils.py @@ -21,7 +21,6 @@ from selenium.common.exceptions import ( WebDriverException, ) from selenium.webdriver.common.by import By -from selenium.webdriver.common.desired_capabilities import DesiredCapabilities from selenium.webdriver.common.keys import Keys from selenium.webdriver.remote.webdriver import WebDriver from selenium.webdriver.remote.webelement import WebElement @@ -33,6 +32,8 @@ from authentik.core.models import User from authentik.managed.manager import ObjectManager RETRIES = int(environ.get("RETRIES", "5")) +BROWSER_NAME = environ.get("BROWSER", "chrome") + # pylint: disable=invalid-name def USER() -> User: # noqa @@ -88,7 +89,11 @@ class SeleniumTestCase(StaticLiveServerTestCase): def _get_driver(self) -> WebDriver: return webdriver.Remote( command_executor="http://localhost:4444/wd/hub", - desired_capabilities=DesiredCapabilities.CHROME, + desired_capabilities={ + "browserName": BROWSER_NAME, + "version": "", + "platform": "ANY", + } ) def tearDown(self):