From 6960adeae64591ad9ab28e7c6f5d6d2ab8892868 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 1 Nov 2016 22:05:13 +0100 Subject: [PATCH 1/9] Add some basic integration tests --- .travis.yml | 8 +++ tests/integration/setup.sh | 5 ++ tests/integration/test_update_scripts.py | 80 ++++++++++++++++++++++++ 3 files changed, 93 insertions(+) create mode 100644 .travis.yml create mode 100644 tests/integration/setup.sh create mode 100644 tests/integration/test_update_scripts.py diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..699810a --- /dev/null +++ b/.travis.yml @@ -0,0 +1,8 @@ +language: python +before_install: +- sudo apt-get update +- sudo apt-get install -y gpg +install: +- sudo sh tests/integration/setup.sh +- python update/test_*.py +- sudo python tests/integration/test_*.py diff --git a/tests/integration/setup.sh b/tests/integration/setup.sh new file mode 100644 index 0000000..4b4ecae --- /dev/null +++ b/tests/integration/setup.sh @@ -0,0 +1,5 @@ +#!/bin/sh +mkdir -p /etc/default/lepidopter +cat << EOF > /etc/default/lepidopter +LEPIDOPTER_BUILD="alpha" +EOF diff --git a/tests/integration/test_update_scripts.py b/tests/integration/test_update_scripts.py new file mode 100644 index 0000000..826549c --- /dev/null +++ b/tests/integration/test_update_scripts.py @@ -0,0 +1,80 @@ +import os +import re +import imp +import time +import operator +import multiprocessing + +import unittest + +UPDATERS_DIR = os.path.abspath( + os.path.join( + os.path.dirname(__file__), + '..', '..', + 'updater', + 'versions' + ) +) + +UPDATER_FILE_REGEXP = re.compile(r"^update-(\d).py$") + +class TestUpdaters(unittest.TestCase): + def setUp(self): + if not os.path.exists("/etc/default/lepidopter"): + raise RuntimeError("These must be run on a system that looks like lepidopter") + + def list_updaters(self): + updaters = [] + for filename in os.listdir(UPDATERS_DIR): + m = UPDATER_FILE_REGEXP.match(filename) + if m: + version = int(m.group(1)) + path = os.path.join(UPDATERS_DIR, filename) + module = imp.load_source( + 'updater_{0}'.format(version), + path + ) + updaters.append({ + 'version': version, + 'module': module, + 'path': path + }) + updaters.sort(key=operator.itemgetter('version')) + return updaters + + def are_versions_consistent(self, updaters): + last_version = 0 + for updater in updaters: + # Ensure versions versions match the filename + self.assertEqual(updater['module'].__version__, + str(updater['version']), + 'Mismatch in __version__ variable of for updater-%s.py' % updater['version']) + # Ensure versions always increase by one + self.assertEqual(last_version + 1, updater['version'], + 'Missing version %s' % last_version + 1) + last_version = updater['version'] + + def updaters_can_be_partial(self, updaters): + for updater in updaters: + p = multiprocessing.Process(target=updater['module'].run) + p.start() + time.sleep(0.5) + p.terminate() + + p = multiprocessing.Process(target=updater['module'].run) + p.start() + p.join() + self.assertEqual(p.exitcode, 0) + + def updaters_smoke_test(self, updaters): + for updater in updaters: + updater['module'].run() + + def test_update_lifecycle(self): + updaters = self.list_updaters() + self.are_versions_consistent(updaters) + self.updaters_can_be_partial(updaters) + self.updaters_smoke_test(updaters) + +if __name__ == "__main__": + unittest.main() From 6093ea2e32b2bcc2d905fa85e588004b101866f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 1 Nov 2016 22:15:41 +0100 Subject: [PATCH 2/9] Add ansible tasks for deploying a test VM to eclipsis --- .gitignore | 2 + deploy/create-vm.yml | 44 +++ deploy/delete-vm.yml | 15 + deploy/library/eclipsis_droplet.py | 318 ++++++++++++++++++ deploy/roles/common/tasks/main.yml | 4 + .../files/lepidopter-install.sh | 96 ++++++ deploy/roles/lepidopter_v0.2.1/tasks/main.yml | 18 + 7 files changed, 497 insertions(+) create mode 100644 deploy/create-vm.yml create mode 100644 deploy/delete-vm.yml create mode 100644 deploy/library/eclipsis_droplet.py create mode 100644 deploy/roles/common/tasks/main.yml create mode 100644 deploy/roles/lepidopter_v0.2.1/files/lepidopter-install.sh create mode 100644 deploy/roles/lepidopter_v0.2.1/tasks/main.yml diff --git a/.gitignore b/.gitignore index 0871533..98c6cb8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ GITHUB_TOKEN *.pyc +*.retry +inventory diff --git a/deploy/create-vm.yml b/deploy/create-vm.yml new file mode 100644 index 0000000..801e86c --- /dev/null +++ b/deploy/create-vm.yml @@ -0,0 +1,44 @@ +- name: ensure that a single instance is launched + hosts: localhost + gather_facts: no + vars: + instance_name: 'lepidopter-update-test' + instance_region: 'ams1' + api_baseurl: "https://portal.eclips.is/portal/api/v2" + ssh_key_id: 77 + tasks: + - name: ensure instance is created + eclipsis_droplet: + state: present + api_baseurl: "{{ api_baseurl }}" + name: "{{ instance_name }}" + region: "{{ instance_region }}" + size: 512 + disk: 10 + ssh_key_id: "{{ ssh_key_id }}" + # This stands for jessie-x64 + image_id: 6 + unique_name: "yes" + register: instance_info + + - debug: + msg: "Instance info {{ instance_info }}" + + - name: add instance to in-memory hosts + add_host: + hostname: "{{ instance_info.droplet.networks.v4[0].ip_address }}" + groupname: eclipsis_hosts + + - name: wait for it to listen on port 22 + wait_for: + state: started + host: "{{ instance_info.droplet.networks.v4[0].ip_address }}" + port: 22 + +- name: ensure that the base updater is setup on the machine + hosts: eclipsis_hosts + remote_user: root + gather_facts: no + roles: + - common + - lepidopter_v0.2.1 diff --git a/deploy/delete-vm.yml b/deploy/delete-vm.yml new file mode 100644 index 0000000..6d00406 --- /dev/null +++ b/deploy/delete-vm.yml @@ -0,0 +1,15 @@ +- name: ensure that the instance does not exist + hosts: localhost + gather_facts: no + vars: + instance_name: 'lepidopter-update-test' + instance_region: 'ams1' + ssh_key_id: 77 + tasks: + - name: ensure instance doesn't exist + eclipsis_droplet: + state: absent + name: "{{ instance_name }}" + unique_name: "yes" + wait_timeout: 10 + register: instance_info diff --git a/deploy/library/eclipsis_droplet.py b/deploy/library/eclipsis_droplet.py new file mode 100644 index 0000000..b2366b0 --- /dev/null +++ b/deploy/library/eclipsis_droplet.py @@ -0,0 +1,318 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# Based off of do_droplet from ansible with modifications to work with the +# eclipsis API. +# requires ansible > 2.0.0 + +import time +import json +from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils.pycompat24 import get_exception +from ansible.module_utils.urls import fetch_url +from ansible.module_utils.basic import env_fallback + +class DropletException(Exception): + pass + +class RestException(Exception): + pass + +class Response(object): + + def __init__(self, resp, info): + self.body = None + if resp: + self.body = resp.read() + self.info = info + + @property + def json(self): + if self.body: + return json.loads(self.body) + elif "body" in self.info: + return json.loads(self.info["body"]) + else: + return None + + @property + def status_code(self): + return self.info["status"] + +class Rest(object): + + def __init__(self, module, headers, baseurl): + self.module = module + self.headers = headers + self.baseurl = baseurl + + def _url_builder(self, path): + if path[0] == '/': + path = path[1:] + return '%s/%s' % (self.baseurl, path) + + def send(self, method, path, data=None, headers=None): + url = self._url_builder(path) + if data is not None: + data = self.module.jsonify(data) + + resp, info = fetch_url(self.module, url, data=data, headers=self.headers, method=method) + + response = Response(resp, info) + + if response.status_code >= 500: + raise RestException('500: Internal server error in ' + '%s: %s %s %s %s' % (method, url, data, self.headers, response.json)) + else: + return response + + def get(self, path, data=None, headers=None): + return self.send('GET', path, data, headers) + + def put(self, path, data=None, headers=None): + return self.send('PUT', path, data, headers) + + def post(self, path, data=None, headers=None): + return self.send('POST', path, data, headers) + + def delete(self, path, data=None, headers=None): + return self.send('DELETE', path, data, headers) + +class DODroplet(object): + + def __init__(self, module): + api_token = module.params['api_token'] + api_baseurl = module.params['api_baseurl'] + self.module = module + self.rest = Rest(module, {'Authorization': 'Bearer {}'.format(api_token), + 'Content-type': 'application/json'}, + api_baseurl) + + def get_key_or_fail(self, k): + v = self.module.params[k] + if v is None: + self.module.fail_json(msg='Unable to load %s' % k) + return v + + def poll_action_for_complete_status(self, action_id): + url = 'actions/{}'.format(action_id) + end_time = time.time() + self.module.params['wait_timeout'] + while time.time() < end_time: + time.sleep(2) + response = self.rest.get(url) + status = response.status_code + json = response.json + if status == 200: + if json['action']['status'].startswith("Instance force stopped."): + return True + if json['action']['status'] == 'completed': + return True + elif json['action']['status'] == 'errored': + raise DropletException('Request to api has failed') + elif status >= 400: + raise DropletException(json['message']) + raise DropletException('Unknown error occured %s' % json) + + def power_on_droplet_request(self, droplet_id): + url = 'droplets/%s/actions' % droplet_id + data = {'type':'power_on'} + response = self.rest.post(url, data=data) + status = response.status_code + json = response.json + if status == 201: + if self.module.params['wait'] == True: + return self.poll_action_for_complete_status(json['action']['id']) + else: + return True + elif status >= 400: + raise DropletException(json['error']) + else: + raise DropletException('Unknown error occured') + + def power_off_droplet_request(self, droplet_id): + url = 'droplets/%s/actions' % droplet_id + data = {'type': 'power_off'} + response = self.rest.post(url, data=data) + status = response.status_code + json = response.json + if status == 201: + if self.module.params['wait'] == True: + return self.poll_action_for_complete_status(json['action']['id']) + else: + return True + elif status >= 400: + raise DropletException(json['error']) + else: + raise DropletException('Unknown error occured') + + def create_droplet_request(self, name, size, image, region, disk, + ssh_key_id): + data = { + "name": name, + "size": size, + "disk": disk, + "image": image, + "region": region, + "ssh_keys": ssh_key_id, + } + response = self.rest.post('/droplet', data=data) + status = response.status_code + json = response.json + if status == 202: + droplet = json['droplet'] + if self.module.params['wait'] == True: + droplet_status = droplet['status'] + droplet_id = droplet['id'] + while droplet_status != 'running': + time.sleep(2) + droplet = self.find_droplet_request(id=droplet_id) + droplet_status = droplet['status'] + return droplet + elif status >= 400: + raise DropletException(json['error']) + else: + raise DropletException('Unknown error occured') + + def delete_droplet_request(self, droplet_id): + url = 'droplets/%s' % droplet_id + response = self.rest.delete(url) + status = response.status_code + json = response.json + if status == 204: + return True + elif status == 404: + return False + elif status >=400: + raise DropletException(json['error']) + else: + raise DropletException('Unknown error occured') + + def find_droplet_request(self, id=None, name=None): + if id is not None: + url = 'droplets/%s' % id + response = self.rest.get(url) + status = response.status_code + json = response.json + if status == 200: + return json['droplet'] + elif status == 404: + return None + elif status >= 400: + raise DropletException(json['error']) + else: + raise DropletException('Unknown error occured') + elif name is not None: + url = 'droplets'; + response = self.rest.get(url) + status = response.status_code + json = response.json + if status == 200: + for droplet in json['droplets']: + if droplet['name']==name: + return droplet + return None + elif status >= 400: + raise DropletException(json['message']) + else: + raise DropletException('Unknown error occured') + else: + return None + + def create_droplet(self): + droplet = None + changed = False + if self.module.params['unique_name']: + droplet = self.find_droplet_request(name=self.get_key_or_fail('name')) + else: + droplet = self.find_droplet_request(id=self.module.params['id']) + if droplet is None: + droplet = self.create_droplet_request( + name=self.get_key_or_fail('name'), + region=self.get_key_or_fail('region'), + size=self.get_key_or_fail('size'), + disk=self.get_key_or_fail('disk'), + image=self.get_key_or_fail('image'), + ssh_key_id=self.module.params['ssh_key_id'], + ) + changed = True + else: + if droplet['status'] == 'off': + changed = self.power_on_droplet_request(droplet['id']) + droplet = self.find_droplet_request(id=droplet['id']) + elif droplet['status'] == 'archive': + raise DropletException('Droplet is in archive state') + self.module.exit_json(changed=changed, droplet=droplet) + + def delete_droplet(self): + droplet_id = None + changed = False + if self.module.params['unique_name']: + droplet = self.find_droplet_request(name=self.get_key_or_fail('name')) + if droplet is not None: + droplet_id = droplet['id'] + else: + droplet_id = self.get_key_or_fail('id') + droplet = self.find_droplet_request(id=droplet_id) + + if droplet['status'] == 'running': + self.power_off_droplet_request(droplet_id) + + if droplet_id is not None: + changed = self.delete_droplet_request(droplet_id) + + self.module.exit_json(changed=changed) + +def core(module): + state = module.params['state'] + droplet = DODroplet(module) + if state == 'present': + droplet.create_droplet() + elif state == 'absent': + droplet.delete_droplet() + +def main(): + module = AnsibleModule( + argument_spec=dict( + state=dict(choices=['present', 'absent'], required=True), + api_token = dict( + aliases=['API_TOKEN'], + no_log=True, + fallback=(env_fallback, ['DO_API_TOKEN', 'DO_API_KEY']), + required=True + ), + api_baseurl=dict(type='str', default="https://portal.eclips.is/portal/api/v2"), + name=dict(type='str'), + unique_name=dict(type="bool", default='no'), + size=dict(aliases=['size_id']), + disk=dict(aliases=['disk_size']), + image=dict(type='int', aliases=['image_id']), + region=dict(aliases=['region_id']), + wait=dict(type="bool", default=True), + wait_timeout=dict(type="int", default=60), + ssh_key_id=dict(type='int') + ), + required_together=( + ['size', 'image', 'region', 'disk'], + ), + required_one_of=( + ['id', 'name'], + ), + ) + + try: + core(module) + except DropletException: + e = get_exception() + module.fail_json(msg=e.message) + except RestException: + e = get_exception() + module.fail_json(msg=e.message) + except KeyError: + e = get_exception() + module.fail_json(msg='Unable to load %s' % e.message) + except Exception: + e = get_exception() + module.fail_json(msg=e.message) + +if __name__ == '__main__': + main() diff --git a/deploy/roles/common/tasks/main.yml b/deploy/roles/common/tasks/main.yml new file mode 100644 index 0000000..6cde848 --- /dev/null +++ b/deploy/roles/common/tasks/main.yml @@ -0,0 +1,4 @@ +- name: update the apt cache + raw: apt-get update +- name: bootstrap the system by installing python and python-simplejson + raw: apt-get install -y python python-simplejson diff --git a/deploy/roles/lepidopter_v0.2.1/files/lepidopter-install.sh b/deploy/roles/lepidopter_v0.2.1/files/lepidopter-install.sh new file mode 100644 index 0000000..6edcd4c --- /dev/null +++ b/deploy/roles/lepidopter_v0.2.1/files/lepidopter-install.sh @@ -0,0 +1,96 @@ +#!/bin/sh +# This file was edited also manually. + +export DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true +export LC_ALL=C LANGUAGE=C LANG=C + +# These were added +LEPIDOPTER_REPO_PATH=/opt/lepidopter/ +export DEB_RELEASE="jessie" +export APT_MIRROR="http://httpredir.debian.org/debian" +export ARCH="amd64" +export MIRROR="http://httpredir.debian.org/debian" +export HOSTNAME_IMG="lepidopter" + +set -x +set -e # Exit on first error + +apt-get install -y netbase ntp less openssh-server git-core \ + binutils ca-certificates wget kmod curl \ + haveged lsb-release tcpdump +# XXX disable localepurge as the interactive setup creates problems +# localepurge +#ROOTDIR="$1" + +# Install non-free binary blob and kernel needed to boot Raspberry Pi +# wget https://raw.github.com/Hexxeh/rpi-update/master/rpi-update \ +# -O ${ROOTDIR}/usr/local/sbin/rpi-update +# chmod a+x ${ROOTDIR}/usr/local/sbin/rpi-update +# mkdir -p ${ROOTDIR}/lib/modules +# touch ${ROOTDIR}/boot/start.elf +# Do not try to update itself, avoid making backups, skip warnings +# export UPDATE_SELF=0 SKIP_BACKUP=1 SKIP_WARNING=1 +# chroot ${ROOTDIR} rpi-update + +# Add module for hardware RNG +# cat <> ${ROOTDIR}/etc/modules +# bcm2708_rng +# EOF + +# Add an apt repository with apt preferences +set_apt_sources() { + SUITE="$1" + PIN_PRIORITY="$2" + COMPONENTS="main" + cat <> /etc/apt/sources.list +# Repository: $SUITE +deb $APT_MIRROR $SUITE $COMPONENTS +EOF + if [ -n "$PIN_PRIORITY" ] + then + cat < /etc/apt/preferences.d/${SUITE}.pref +Package: * +Pin: release n=$SUITE +Pin-Priority: $PIN_PRIORITY +EOF + fi +} + +echo "Add Debian ${DEB_RELEASE}-backports repository" +set_apt_sources ${DEB_RELEASE}-backports +echo "Add Debian stretch repository" +set_apt_sources stretch 100 + +# Create ooniprobe log directory +mkdir -p /var/log/ooni/ + +# Copy required scripts, cronjobs and config files to lepidopter +# Rsync Directory/file hieratchy to image +rsync -avp $LEPIDOPTER_REPO_PATH/lepidopter-fh/ / + +# Install ooniprobe via setup script +/setup-ooniprobe.sh +rm /setup-ooniprobe.sh + +# Execute cleanup script +/cleanup.sh +rm /cleanup.sh + +# Remove SSH host keys and add regenerate_ssh_host_keys SYSV script +#/remove_ssh_host_keys.sh +# Don't regenerate ssh keys +rm /etc/init.d/regenerate_ssh_host_keys +rm /remove_ssh_host_keys.sh + +# Remove motd file and create symlink for lepidopter dynamic MOTD +rm /etc/motd +ln -s /var/run/motd /etc/motd + +# Add (optional) pluggable transport support in tor config +cat $LEPIDOPTER_REPO_PATH/conf/tor-pt.conf >> /etc/tor/torrc + +useradd -G sudo lepidopter +echo "lepidopter:lepidopter"|chpasswd + +echo "Customize script finished successfully." +exit 0 diff --git a/deploy/roles/lepidopter_v0.2.1/tasks/main.yml b/deploy/roles/lepidopter_v0.2.1/tasks/main.yml new file mode 100644 index 0000000..bc47111 --- /dev/null +++ b/deploy/roles/lepidopter_v0.2.1/tasks/main.yml @@ -0,0 +1,18 @@ +--- +- name: ensure required packages installed + apt: name={{ item }} state=present + with_items: + - git + - python-pip + - sudo + +- name: clone lepidopter repository + git: repo=https://github.com/TheTorProject/lepidopter.git + dest=/opt/lepidopter + version=v0.2.1-alpha + +- name: run setup script for lepidopter + script: lepidopter-install.sh + +- name: run ooniprobe update to trigger running the updater to the latest version + command: sh /opt/ooni/update-ooniprobe.sh From 61c8a51f742e0c3c994f5f4b08d0c1093728e758 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 1 Nov 2016 22:15:55 +0100 Subject: [PATCH 3/9] Update travis to use makefile --- .travis.yml | 12 +++++++----- Makefile | 19 +++++++++++++++++++ tests/integration/setup.sh | 2 +- tests/integration/test_update_scripts.py | 13 ++++++++++++- updater/test_updater.py | 1 + 5 files changed, 40 insertions(+), 7 deletions(-) create mode 100644 Makefile diff --git a/.travis.yml b/.travis.yml index 699810a..f21365d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,10 @@ language: python +python: + - "2.7" before_install: -- sudo apt-get update -- sudo apt-get install -y gpg + - sudo apt-get update install: -- sudo sh tests/integration/setup.sh -- python update/test_*.py -- sudo python tests/integration/test_*.py + - sudo make setupintegration +script: + - sudo make integrationtest + - make unittest diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..c2c5623 --- /dev/null +++ b/Makefile @@ -0,0 +1,19 @@ +all: + echo "nothing" + +setupintegration: + sh tests/integration/setup.sh + +integrationtest: + python tests/integration/test_*.py + +unittest: + python updater/test_*.py + +create-testvm: + ansible-playbook -i deploy/inventory deploy/create-vm.yml + +delete-testvm: + ansible-playbook -i deploy/inventory deploy/delete-vm.yml + +reset-testvm: delete-testvm create-testvm diff --git a/tests/integration/setup.sh b/tests/integration/setup.sh index 4b4ecae..1875a9d 100644 --- a/tests/integration/setup.sh +++ b/tests/integration/setup.sh @@ -1,5 +1,5 @@ #!/bin/sh -mkdir -p /etc/default/lepidopter +mkdir -p /etc/default cat << EOF > /etc/default/lepidopter LEPIDOPTER_BUILD="alpha" EOF diff --git a/tests/integration/test_update_scripts.py b/tests/integration/test_update_scripts.py index 826549c..34b4f13 100644 --- a/tests/integration/test_update_scripts.py +++ b/tests/integration/test_update_scripts.py @@ -2,7 +2,9 @@ import re import imp import time +import signal import operator +import subprocess import multiprocessing import unittest @@ -18,6 +20,14 @@ UPDATER_FILE_REGEXP = re.compile(r"^update-(\d).py$") +def kill_children(pid, sig=signal.SIGTERM): + p = subprocess.Popen("ps -o pid --ppid %d --noheaders" % pid, + shell=True, stdout=subprocess.PIPE) + output = p.stdout.read() + p.wait() + for child_pid in output.split("\n")[:-1]: + os.kill(int(child_pid), sig) + class TestUpdaters(unittest.TestCase): def setUp(self): if not os.path.exists("/etc/default/lepidopter"): @@ -51,7 +61,7 @@ def are_versions_consistent(self, updaters): 'Mismatch in __version__ variable of for updater-%s.py' % updater['version']) # Ensure versions always increase by one self.assertEqual(last_version + 1, updater['version'], - 'Missing version %s' % last_version + 1) + 'Missing version %s' % (last_version + 1)) last_version = updater['version'] def updaters_can_be_partial(self, updaters): @@ -59,6 +69,7 @@ def updaters_can_be_partial(self, updaters): p = multiprocessing.Process(target=updater['module'].run) p.start() time.sleep(0.5) + kill_children(p.pid) p.terminate() p = multiprocessing.Process(target=updater['module'].run) diff --git a/updater/test_updater.py b/updater/test_updater.py index 7294ff2..1d1e613 100644 --- a/updater/test_updater.py +++ b/updater/test_updater.py @@ -9,6 +9,7 @@ class TestSecurity(unittest.TestCase): def test_verify_file(self): o = verify_file( os.path.join(CWD, "versions", "update-1.py.asc"), + os.path.join(CWD, "versions", "update-1.py"), os.path.join(CWD, "public.asc") ) print o From 800d7178386fc2060d09ea117272141f59a2cbc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 2 Nov 2016 00:01:46 +0100 Subject: [PATCH 4/9] Add a dockerfile based integration test for lepidopter-updater --- .travis.yml | 9 +++++---- Makefile | 6 ------ tests/integration/Dockerfile | 19 +++++++++++++++++++ .../lepidopter-install-v0.2.1-alpha.sh | 1 + .../{test_update_scripts.py => main.py} | 0 5 files changed, 25 insertions(+), 10 deletions(-) create mode 100644 tests/integration/Dockerfile create mode 120000 tests/integration/lepidopter-install-v0.2.1-alpha.sh rename tests/integration/{test_update_scripts.py => main.py} (100%) diff --git a/.travis.yml b/.travis.yml index f21365d..08cbb6f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,11 @@ +sudo: required +services: + - docker language: python python: - "2.7" before_install: - - sudo apt-get update -install: - - sudo make setupintegration + - docker build -t lepidopter/integration tests/integration script: - - sudo make integrationtest - make unittest + - docker run lepidopter/integration diff --git a/Makefile b/Makefile index c2c5623..afb6f59 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,6 @@ all: echo "nothing" -setupintegration: - sh tests/integration/setup.sh - -integrationtest: - python tests/integration/test_*.py - unittest: python updater/test_*.py diff --git a/tests/integration/Dockerfile b/tests/integration/Dockerfile new file mode 100644 index 0000000..7b13534 --- /dev/null +++ b/tests/integration/Dockerfile @@ -0,0 +1,19 @@ +FROM debian:jessie +ARG lepidopter_tag="v0.2.1-alpha" + +RUN apt-get update && apt-get install -y \ + git \ + python-pip \ + sudo + +RUN git clone -b $lepidopter_tag https://github.com/TheTorProject/lepidopter.git /opt/lepidopter/ +ADD lepidopter-install-${lepidopter_tag}.sh +RUN lepidopter-install-${lepidopter_tag}.sh + +RUN service lepidopter-updater stop + +RUN mkdir -p /integration +WORKDIR /integration +COPY main.py /integration/ + +CMD [ "python", "main.py" ] diff --git a/tests/integration/lepidopter-install-v0.2.1-alpha.sh b/tests/integration/lepidopter-install-v0.2.1-alpha.sh new file mode 120000 index 0000000..660bb54 --- /dev/null +++ b/tests/integration/lepidopter-install-v0.2.1-alpha.sh @@ -0,0 +1 @@ +../../deploy/roles/lepidopter_v0.2.1/files/lepidopter-install.sh \ No newline at end of file diff --git a/tests/integration/test_update_scripts.py b/tests/integration/main.py similarity index 100% rename from tests/integration/test_update_scripts.py rename to tests/integration/main.py From d5c9e7c68888fc34469743420006c80781361fb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 2 Nov 2016 12:55:51 +0100 Subject: [PATCH 5/9] Fix docker based build --- tests/integration/Dockerfile | 9 +- .../lepidopter-install-v0.2.1-alpha.sh | 97 ++++++++++++++++++- 2 files changed, 101 insertions(+), 5 deletions(-) mode change 120000 => 100755 tests/integration/lepidopter-install-v0.2.1-alpha.sh diff --git a/tests/integration/Dockerfile b/tests/integration/Dockerfile index 7b13534..d67b456 100644 --- a/tests/integration/Dockerfile +++ b/tests/integration/Dockerfile @@ -7,13 +7,14 @@ RUN apt-get update && apt-get install -y \ sudo RUN git clone -b $lepidopter_tag https://github.com/TheTorProject/lepidopter.git /opt/lepidopter/ -ADD lepidopter-install-${lepidopter_tag}.sh -RUN lepidopter-install-${lepidopter_tag}.sh - -RUN service lepidopter-updater stop RUN mkdir -p /integration WORKDIR /integration + +COPY lepidopter-install-${lepidopter_tag}.sh /integration +RUN /integration/lepidopter-install-${lepidopter_tag}.sh +RUN service lepidopter-updater stop + COPY main.py /integration/ CMD [ "python", "main.py" ] diff --git a/tests/integration/lepidopter-install-v0.2.1-alpha.sh b/tests/integration/lepidopter-install-v0.2.1-alpha.sh deleted file mode 120000 index 660bb54..0000000 --- a/tests/integration/lepidopter-install-v0.2.1-alpha.sh +++ /dev/null @@ -1 +0,0 @@ -../../deploy/roles/lepidopter_v0.2.1/files/lepidopter-install.sh \ No newline at end of file diff --git a/tests/integration/lepidopter-install-v0.2.1-alpha.sh b/tests/integration/lepidopter-install-v0.2.1-alpha.sh new file mode 100755 index 0000000..6edcd4c --- /dev/null +++ b/tests/integration/lepidopter-install-v0.2.1-alpha.sh @@ -0,0 +1,96 @@ +#!/bin/sh +# This file was edited also manually. + +export DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true +export LC_ALL=C LANGUAGE=C LANG=C + +# These were added +LEPIDOPTER_REPO_PATH=/opt/lepidopter/ +export DEB_RELEASE="jessie" +export APT_MIRROR="http://httpredir.debian.org/debian" +export ARCH="amd64" +export MIRROR="http://httpredir.debian.org/debian" +export HOSTNAME_IMG="lepidopter" + +set -x +set -e # Exit on first error + +apt-get install -y netbase ntp less openssh-server git-core \ + binutils ca-certificates wget kmod curl \ + haveged lsb-release tcpdump +# XXX disable localepurge as the interactive setup creates problems +# localepurge +#ROOTDIR="$1" + +# Install non-free binary blob and kernel needed to boot Raspberry Pi +# wget https://raw.github.com/Hexxeh/rpi-update/master/rpi-update \ +# -O ${ROOTDIR}/usr/local/sbin/rpi-update +# chmod a+x ${ROOTDIR}/usr/local/sbin/rpi-update +# mkdir -p ${ROOTDIR}/lib/modules +# touch ${ROOTDIR}/boot/start.elf +# Do not try to update itself, avoid making backups, skip warnings +# export UPDATE_SELF=0 SKIP_BACKUP=1 SKIP_WARNING=1 +# chroot ${ROOTDIR} rpi-update + +# Add module for hardware RNG +# cat <> ${ROOTDIR}/etc/modules +# bcm2708_rng +# EOF + +# Add an apt repository with apt preferences +set_apt_sources() { + SUITE="$1" + PIN_PRIORITY="$2" + COMPONENTS="main" + cat <> /etc/apt/sources.list +# Repository: $SUITE +deb $APT_MIRROR $SUITE $COMPONENTS +EOF + if [ -n "$PIN_PRIORITY" ] + then + cat < /etc/apt/preferences.d/${SUITE}.pref +Package: * +Pin: release n=$SUITE +Pin-Priority: $PIN_PRIORITY +EOF + fi +} + +echo "Add Debian ${DEB_RELEASE}-backports repository" +set_apt_sources ${DEB_RELEASE}-backports +echo "Add Debian stretch repository" +set_apt_sources stretch 100 + +# Create ooniprobe log directory +mkdir -p /var/log/ooni/ + +# Copy required scripts, cronjobs and config files to lepidopter +# Rsync Directory/file hieratchy to image +rsync -avp $LEPIDOPTER_REPO_PATH/lepidopter-fh/ / + +# Install ooniprobe via setup script +/setup-ooniprobe.sh +rm /setup-ooniprobe.sh + +# Execute cleanup script +/cleanup.sh +rm /cleanup.sh + +# Remove SSH host keys and add regenerate_ssh_host_keys SYSV script +#/remove_ssh_host_keys.sh +# Don't regenerate ssh keys +rm /etc/init.d/regenerate_ssh_host_keys +rm /remove_ssh_host_keys.sh + +# Remove motd file and create symlink for lepidopter dynamic MOTD +rm /etc/motd +ln -s /var/run/motd /etc/motd + +# Add (optional) pluggable transport support in tor config +cat $LEPIDOPTER_REPO_PATH/conf/tor-pt.conf >> /etc/tor/torrc + +useradd -G sudo lepidopter +echo "lepidopter:lepidopter"|chpasswd + +echo "Customize script finished successfully." +exit 0 From a55500d98da1ed126890261602d880e61c5c7e68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 2 Nov 2016 13:10:29 +0100 Subject: [PATCH 6/9] Fix the version of ooniprobe installed in lepidopter 0.2.1 --- tests/integration/lepidopter-install-v0.2.1-alpha.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration/lepidopter-install-v0.2.1-alpha.sh b/tests/integration/lepidopter-install-v0.2.1-alpha.sh index 6edcd4c..fecda3b 100755 --- a/tests/integration/lepidopter-install-v0.2.1-alpha.sh +++ b/tests/integration/lepidopter-install-v0.2.1-alpha.sh @@ -68,6 +68,9 @@ mkdir -p /var/log/ooni/ # Rsync Directory/file hieratchy to image rsync -avp $LEPIDOPTER_REPO_PATH/lepidopter-fh/ / +# Install the version of ooniprobe in alpha +pip install ooniprobe=="1.4.1" + # Install ooniprobe via setup script /setup-ooniprobe.sh rm /setup-ooniprobe.sh From e3897ea59f42693d475fe5da192ac01d9ea4be53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 2 Nov 2016 13:23:17 +0100 Subject: [PATCH 7/9] Dependencies for ooniprobe --- tests/integration/lepidopter-install-v0.2.1-alpha.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/lepidopter-install-v0.2.1-alpha.sh b/tests/integration/lepidopter-install-v0.2.1-alpha.sh index fecda3b..bca70f9 100755 --- a/tests/integration/lepidopter-install-v0.2.1-alpha.sh +++ b/tests/integration/lepidopter-install-v0.2.1-alpha.sh @@ -69,6 +69,8 @@ mkdir -p /var/log/ooni/ rsync -avp $LEPIDOPTER_REPO_PATH/lepidopter-fh/ / # Install the version of ooniprobe in alpha +apt-get -y install openssl libssl-dev libyaml-dev libffi-dev libpcap-dev tor \ + libgeoip-dev libdumbnet-dev python-dev python-pip libgmp-dev pip install ooniprobe=="1.4.1" # Install ooniprobe via setup script From c8ec8244cab5189b153ba83c0001bf0c4cd062ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 2 Nov 2016 15:42:43 +0100 Subject: [PATCH 8/9] Lepidopter service is not installed. No need to stop it --- tests/integration/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/Dockerfile b/tests/integration/Dockerfile index d67b456..39991a9 100644 --- a/tests/integration/Dockerfile +++ b/tests/integration/Dockerfile @@ -13,7 +13,6 @@ WORKDIR /integration COPY lepidopter-install-${lepidopter_tag}.sh /integration RUN /integration/lepidopter-install-${lepidopter_tag}.sh -RUN service lepidopter-updater stop COPY main.py /integration/ From d3e30f582ab5304a8381e88980b508cf2204fa39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 2 Nov 2016 16:15:40 +0100 Subject: [PATCH 9/9] Clone the lepidopter update repo inside of the docker --- .travis.yml | 2 +- tests/integration/Dockerfile | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 08cbb6f..31fbfdc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ language: python python: - "2.7" before_install: - - docker build -t lepidopter/integration tests/integration + - docker build -t lepidopter/integration --build-arg lepidopter_update_tag=$TRAVIS_COMMIT tests/integration script: - make unittest - docker run lepidopter/integration diff --git a/tests/integration/Dockerfile b/tests/integration/Dockerfile index 39991a9..4449fe6 100644 --- a/tests/integration/Dockerfile +++ b/tests/integration/Dockerfile @@ -1,5 +1,6 @@ FROM debian:jessie ARG lepidopter_tag="v0.2.1-alpha" +ARG lepidopter_update_tag="master" RUN apt-get update && apt-get install -y \ git \ @@ -11,9 +12,7 @@ RUN git clone -b $lepidopter_tag https://github.com/TheTorProject/lepidopter.git RUN mkdir -p /integration WORKDIR /integration -COPY lepidopter-install-${lepidopter_tag}.sh /integration -RUN /integration/lepidopter-install-${lepidopter_tag}.sh +RUN git clone -b $lepidopter_update_tag https://github.com/OpenObservatory/lepidopter-update.git /integration/lepidopter-update +RUN lepidopter-update/tests/integration/lepidopter-install-${lepidopter_tag}.sh -COPY main.py /integration/ - -CMD [ "python", "main.py" ] +CMD [ "python", "lepidopter-update/tests/integration/main.py" ]