From 3b6119161432c7498ba32aa9cf45248d8677eb12 Mon Sep 17 00:00:00 2001 From: Jens Langhammer Date: Mon, 5 Oct 2020 23:11:44 +0200 Subject: [PATCH] outpost: enable docker controller --- docker-compose.yml | 1 + passbook/core/models.py | 4 +- passbook/outposts/controllers/docker.py | 78 ++++++++++++++++++++----- passbook/outposts/settings.py | 16 ++--- 4 files changed, 72 insertions(+), 27 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 45039345d..afca41a68 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -52,6 +52,7 @@ services: PASSBOOK_POSTGRESQL__PASSWORD: ${PG_PASS} volumes: - ./backups:/backups + - /var/run/docker.socket:/var/run/docker.socket env_file: - .env static: diff --git a/passbook/core/models.py b/passbook/core/models.py index dac0b3ede..1e8bc0752 100644 --- a/passbook/core/models.py +++ b/passbook/core/models.py @@ -313,9 +313,7 @@ class Token(ExpiringModel): description = models.TextField(default="", blank=True) def __str__(self): - return ( - f"Token {self.identifier} (expires={self.expires})" - ) + return f"Token {self.identifier} (expires={self.expires})" class Meta: diff --git a/passbook/outposts/controllers/docker.py b/passbook/outposts/controllers/docker.py index 3f1ab8ecc..fd133c360 100644 --- a/passbook/outposts/controllers/docker.py +++ b/passbook/outposts/controllers/docker.py @@ -1,4 +1,6 @@ """Docker controller""" +from typing import Dict, Tuple + from docker import DockerClient, from_env from docker.errors import NotFound from docker.models.containers import Container @@ -21,36 +23,80 @@ class DockerController(BaseController): super().__init__(outpost_pk) self.client = from_env() - def _get_container(self) -> Container: + def _get_env(self) -> Dict[str, str]: + return { + "PASSBOOK_HOST": self.outpost.config.passbook_host, + "PASSBOOK_INSECURE": str(self.outpost.config.passbook_host_insecure), + "PASSBOOK_TOKEN": self.outpost.token.token_uuid.hex, + } + + def _comp_env(self, container: Container) -> bool: + """Check if container's env is equal to what we would set. Return true if container needs + to be rebuilt.""" + should_be = self._get_env() + container_env = container.attrs.get("Config", {}).get("Env", {}) + for key, expected_value in should_be.items(): + if key not in container_env: + continue + if container_env[key] != expected_value: + return True + return False + + def _get_container(self) -> Tuple[Container, bool]: container_name = f"passbook-proxy-{self.outpost.uuid.hex}" try: - return self.client.containers.get(container_name) + return self.client.containers.get(container_name), False except NotFound: - return self.client.containers.create( - image=f"{self.image_base}-{self.outpost.type}:{__version__}", - name=f"passbook-proxy-{self.outpost.uuid.hex}", - detach=True, - ports={x: x for _, x in self.deployment_ports.items()}, - environment={ - "PASSBOOK_HOST": self.outpost.config.passbook_host, - "PASSBOOK_INSECURE": str( - self.outpost.config.passbook_host_insecure - ), - "PASSBOOK_TOKEN": self.outpost.token.token_uuid.hex, - }, + self.logger.info("Container does not exist, creating") + image_name = f"{self.image_base}-{self.outpost.type}:{__version__}" + self.client.images.pull(image_name) + return ( + self.client.containers.create( + image=image_name, + name=f"passbook-proxy-{self.outpost.uuid.hex}", + detach=True, + ports={x: x for _, x in self.deployment_ports.items()}, + environment=self._get_env(), + ), + True, ) def run(self): - container = self._get_container() + container, has_been_created = self._get_container() + if has_been_created: + return None # Check if the container is out of date, delete it and retry if len(container.image.tags) > 0: - tag: str = container.iamge.tags[0] + tag: str = container.image.tags[0] _, _, version = tag.partition(":") if version != __version__: + self.logger.info( + "Container has mismatched version, re-creating...", + has=version, + should=__version__, + ) container.kill() container.remove(force=True) return self.run() + # Check that container values match our values + if self._comp_env(container): + self.logger.info("Container has outdated config, re-creating...") + container.kill() + container.remove(force=True) + return self.run() + # Check that container is healthy + if ( + container.status == "running" + and container.attrs.get("State", {}).get("Health", {}).get("Status", "") + != "healthy" + ): + # At this point we know the config is correct, but the container isn't healthy, + # so we just restart it with the same config + self.logger.info("Container is unhealthy, restarting...") + container.restart() + # Check that container is running if container.status != "running": + self.logger.info("Container is not running, restarting...") container.start() return None diff --git a/passbook/outposts/settings.py b/passbook/outposts/settings.py index d3f8a9b0b..62589e1c9 100644 --- a/passbook/outposts/settings.py +++ b/passbook/outposts/settings.py @@ -1,10 +1,10 @@ """Outposts Settings""" -# from celery.schedules import crontab +from celery.schedules import crontab -# CELERY_BEAT_SCHEDULE = { -# "outposts_k8s": { -# "task": "passbook.outposts.tasks.outpost_k8s_controller", -# "schedule": crontab(minute="*/5"), # Run every 5 minutes -# "options": {"queue": "passbook_scheduled"}, -# } -# } +CELERY_BEAT_SCHEDULE = { + "outposts_k8s": { + "task": "passbook.outposts.tasks.outpost_controller", + "schedule": crontab(minute="*/5"), # Run every 5 minutes + "options": {"queue": "passbook_scheduled"}, + } +}