From 7f0cbc11e1831c6545215354314522cceb2965bd Mon Sep 17 00:00:00 2001 From: Pavel Tikhomirov Date: Fri, 6 Feb 2015 18:31:16 +0300 Subject: [PATCH] destroy_containers: wait containers to start if use vfs graphdriver for docker, we can start cleanup before all docker-containers are started, so can leave behind some ct Signed-off-by: Pavel Tikhomirov --- docker-stress | 2 +- spotify/docker_stress/docker_client.py | 13 +++++++++++-- spotify/docker_stress/docker_util.py | 21 +++++++++++++++++---- 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/docker-stress b/docker-stress index cfcc3db..126d41a 100755 --- a/docker-stress +++ b/docker-stress @@ -138,7 +138,7 @@ def main(): except KeyboardInterrupt: pass finally: - print "Terminating workers and cleaning up..." + print "Terminating workers and cleaning up...(in background)" kill_workers() destroy_containers(client, nametag) diff --git a/spotify/docker_stress/docker_client.py b/spotify/docker_stress/docker_client.py index f32f3ae..134bc21 100644 --- a/spotify/docker_stress/docker_client.py +++ b/spotify/docker_stress/docker_client.py @@ -97,11 +97,20 @@ def destroy(self, container_id): log.debug('destroy %s', container_id) self.cli_check('rm', container_id) - def list_containers(self, needle=''): + def list_containers(self, needle='', _all=False): if not needle: return self.cli_check('ps', '-q').splitlines() else: - lines = self.cli_check('ps').splitlines()[1:] + if _all: + lines = self.cli_check('ps', '-a').splitlines()[1:] + else: + lines = self.cli_check('ps').splitlines()[1:] matches = [word for line in lines for word in line.split() if needle in word] log.debug('list_containers: needle=%s, matches=%s', needle, matches) return matches + + def list_all_exited(self, needle): + lines = self.cli_check('ps', '-a').splitlines()[1:] + lines = [line for line in lines if 'Exited' in line] + matches = [word for line in lines for word in line.split() if needle in word] + return matches diff --git a/spotify/docker_stress/docker_util.py b/spotify/docker_stress/docker_util.py index c04c304..1d65005 100644 --- a/spotify/docker_stress/docker_util.py +++ b/spotify/docker_stress/docker_util.py @@ -1,6 +1,7 @@ +import os import logging from socket import create_connection -from time import sleep +from time import time, sleep from urlparse import urlparse from docker_client import DockerClientError @@ -49,22 +50,34 @@ def connectable(client, container_id, hostname, ports): def destroy_containers(client, nametag): - while True: + timeout = time() + 60 * 60 + try: + if os.fork(): + return + except OSError, e: + log.debug('failed to fork: %s', e) + return + + while time() < timeout: sleep(1) - containers = client.list_containers(needle=nametag) + containers = client.list_containers(needle=nametag, _all=True) if not containers: break + containers = client.list_containers(needle=nametag) for container in containers: try: client.kill(container) except Exception, e: log.debug('kill failed: %s', e) + containers = client.list_all_exited(needle=nametag) for container in containers: try: client.destroy(container) except Exception, e: log.debug('destroy failed: %s', e) - + sleep(99) + if time() > timeout: + log.debug('Reached timeout for container destroy') def endpoint_address(endpoint): if '://' not in endpoint: