diff --git a/.gitignore b/.gitignore index 0871533..98c6cb8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ GITHUB_TOKEN *.pyc +*.retry +inventory diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..31fbfdc --- /dev/null +++ b/.travis.yml @@ -0,0 +1,11 @@ +sudo: required +services: + - docker +language: python +python: + - "2.7" +before_install: + - docker build -t lepidopter/integration --build-arg lepidopter_update_tag=$TRAVIS_COMMIT tests/integration +script: + - make unittest + - docker run lepidopter/integration diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..afb6f59 --- /dev/null +++ b/Makefile @@ -0,0 +1,13 @@ +all: + echo "nothing" + +unittest: + python updater/test_*.py + +create-testvm: + ansible-playbook -i deploy/inventory deploy/create-vm.yml + +delete-testvm: + ansible-playbook -i deploy/inventory deploy/delete-vm.yml + +reset-testvm: delete-testvm create-testvm diff --git a/deploy/create-vm.yml b/deploy/create-vm.yml new file mode 100644 index 0000000..801e86c --- /dev/null +++ b/deploy/create-vm.yml @@ -0,0 +1,44 @@ +- name: ensure that a single instance is launched + hosts: localhost + gather_facts: no + vars: + instance_name: 'lepidopter-update-test' + instance_region: 'ams1' + api_baseurl: "https://portal.eclips.is/portal/api/v2" + ssh_key_id: 77 + tasks: + - name: ensure instance is created + eclipsis_droplet: + state: present + api_baseurl: "{{ api_baseurl }}" + name: "{{ instance_name }}" + region: "{{ instance_region }}" + size: 512 + disk: 10 + ssh_key_id: "{{ ssh_key_id }}" + # This stands for jessie-x64 + image_id: 6 + unique_name: "yes" + register: instance_info + + - debug: + msg: "Instance info {{ instance_info }}" + + - name: add instance to in-memory hosts + add_host: + hostname: "{{ instance_info.droplet.networks.v4[0].ip_address }}" + groupname: eclipsis_hosts + + - name: wait for it to listen on port 22 + wait_for: + state: started + host: "{{ instance_info.droplet.networks.v4[0].ip_address }}" + port: 22 + +- name: ensure that the base updater is setup on the machine + hosts: eclipsis_hosts + remote_user: root + gather_facts: no + roles: + - common + - lepidopter_v0.2.1 diff --git a/deploy/delete-vm.yml b/deploy/delete-vm.yml new file mode 100644 index 0000000..6d00406 --- /dev/null +++ b/deploy/delete-vm.yml @@ -0,0 +1,15 @@ +- name: ensure that the instance does not exist + hosts: localhost + gather_facts: no + vars: + instance_name: 'lepidopter-update-test' + instance_region: 'ams1' + ssh_key_id: 77 + tasks: + - name: ensure instance doesn't exist + eclipsis_droplet: + state: absent + name: "{{ instance_name }}" + unique_name: "yes" + wait_timeout: 10 + register: instance_info diff --git a/deploy/library/eclipsis_droplet.py b/deploy/library/eclipsis_droplet.py new file mode 100644 index 0000000..b2366b0 --- /dev/null +++ b/deploy/library/eclipsis_droplet.py @@ -0,0 +1,318 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# Based off of do_droplet from ansible with modifications to work with the +# eclipsis API. +# requires ansible > 2.0.0 + +import time +import json +from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils.pycompat24 import get_exception +from ansible.module_utils.urls import fetch_url +from ansible.module_utils.basic import env_fallback + +class DropletException(Exception): + pass + +class RestException(Exception): + pass + +class Response(object): + + def __init__(self, resp, info): + self.body = None + if resp: + self.body = resp.read() + self.info = info + + @property + def json(self): + if self.body: + return json.loads(self.body) + elif "body" in self.info: + return json.loads(self.info["body"]) + else: + return None + + @property + def status_code(self): + return self.info["status"] + +class Rest(object): + + def __init__(self, module, headers, baseurl): + self.module = module + self.headers = headers + self.baseurl = baseurl + + def _url_builder(self, path): + if path[0] == '/': + path = path[1:] + return '%s/%s' % (self.baseurl, path) + + def send(self, method, path, data=None, headers=None): + url = self._url_builder(path) + if data is not None: + data = self.module.jsonify(data) + + resp, info = fetch_url(self.module, url, data=data, headers=self.headers, method=method) + + response = Response(resp, info) + + if response.status_code >= 500: + raise RestException('500: Internal server error in ' + '%s: %s %s %s %s' % (method, url, data, self.headers, response.json)) + else: + return response + + def get(self, path, data=None, headers=None): + return self.send('GET', path, data, headers) + + def put(self, path, data=None, headers=None): + return self.send('PUT', path, data, headers) + + def post(self, path, data=None, headers=None): + return self.send('POST', path, data, headers) + + def delete(self, path, data=None, headers=None): + return self.send('DELETE', path, data, headers) + +class DODroplet(object): + + def __init__(self, module): + api_token = module.params['api_token'] + api_baseurl = module.params['api_baseurl'] + self.module = module + self.rest = Rest(module, {'Authorization': 'Bearer {}'.format(api_token), + 'Content-type': 'application/json'}, + api_baseurl) + + def get_key_or_fail(self, k): + v = self.module.params[k] + if v is None: + self.module.fail_json(msg='Unable to load %s' % k) + return v + + def poll_action_for_complete_status(self, action_id): + url = 'actions/{}'.format(action_id) + end_time = time.time() + self.module.params['wait_timeout'] + while time.time() < end_time: + time.sleep(2) + response = self.rest.get(url) + status = response.status_code + json = response.json + if status == 200: + if json['action']['status'].startswith("Instance force stopped."): + return True + if json['action']['status'] == 'completed': + return True + elif json['action']['status'] == 'errored': + raise DropletException('Request to api has failed') + elif status >= 400: + raise DropletException(json['message']) + raise DropletException('Unknown error occured %s' % json) + + def power_on_droplet_request(self, droplet_id): + url = 'droplets/%s/actions' % droplet_id + data = {'type':'power_on'} + response = self.rest.post(url, data=data) + status = response.status_code + json = response.json + if status == 201: + if self.module.params['wait'] == True: + return self.poll_action_for_complete_status(json['action']['id']) + else: + return True + elif status >= 400: + raise DropletException(json['error']) + else: + raise DropletException('Unknown error occured') + + def power_off_droplet_request(self, droplet_id): + url = 'droplets/%s/actions' % droplet_id + data = {'type': 'power_off'} + response = self.rest.post(url, data=data) + status = response.status_code + json = response.json + if status == 201: + if self.module.params['wait'] == True: + return self.poll_action_for_complete_status(json['action']['id']) + else: + return True + elif status >= 400: + raise DropletException(json['error']) + else: + raise DropletException('Unknown error occured') + + def create_droplet_request(self, name, size, image, region, disk, + ssh_key_id): + data = { + "name": name, + "size": size, + "disk": disk, + "image": image, + "region": region, + "ssh_keys": ssh_key_id, + } + response = self.rest.post('/droplet', data=data) + status = response.status_code + json = response.json + if status == 202: + droplet = json['droplet'] + if self.module.params['wait'] == True: + droplet_status = droplet['status'] + droplet_id = droplet['id'] + while droplet_status != 'running': + time.sleep(2) + droplet = self.find_droplet_request(id=droplet_id) + droplet_status = droplet['status'] + return droplet + elif status >= 400: + raise DropletException(json['error']) + else: + raise DropletException('Unknown error occured') + + def delete_droplet_request(self, droplet_id): + url = 'droplets/%s' % droplet_id + response = self.rest.delete(url) + status = response.status_code + json = response.json + if status == 204: + return True + elif status == 404: + return False + elif status >=400: + raise DropletException(json['error']) + else: + raise DropletException('Unknown error occured') + + def find_droplet_request(self, id=None, name=None): + if id is not None: + url = 'droplets/%s' % id + response = self.rest.get(url) + status = response.status_code + json = response.json + if status == 200: + return json['droplet'] + elif status == 404: + return None + elif status >= 400: + raise DropletException(json['error']) + else: + raise DropletException('Unknown error occured') + elif name is not None: + url = 'droplets'; + response = self.rest.get(url) + status = response.status_code + json = response.json + if status == 200: + for droplet in json['droplets']: + if droplet['name']==name: + return droplet + return None + elif status >= 400: + raise DropletException(json['message']) + else: + raise DropletException('Unknown error occured') + else: + return None + + def create_droplet(self): + droplet = None + changed = False + if self.module.params['unique_name']: + droplet = self.find_droplet_request(name=self.get_key_or_fail('name')) + else: + droplet = self.find_droplet_request(id=self.module.params['id']) + if droplet is None: + droplet = self.create_droplet_request( + name=self.get_key_or_fail('name'), + region=self.get_key_or_fail('region'), + size=self.get_key_or_fail('size'), + disk=self.get_key_or_fail('disk'), + image=self.get_key_or_fail('image'), + ssh_key_id=self.module.params['ssh_key_id'], + ) + changed = True + else: + if droplet['status'] == 'off': + changed = self.power_on_droplet_request(droplet['id']) + droplet = self.find_droplet_request(id=droplet['id']) + elif droplet['status'] == 'archive': + raise DropletException('Droplet is in archive state') + self.module.exit_json(changed=changed, droplet=droplet) + + def delete_droplet(self): + droplet_id = None + changed = False + if self.module.params['unique_name']: + droplet = self.find_droplet_request(name=self.get_key_or_fail('name')) + if droplet is not None: + droplet_id = droplet['id'] + else: + droplet_id = self.get_key_or_fail('id') + droplet = self.find_droplet_request(id=droplet_id) + + if droplet['status'] == 'running': + self.power_off_droplet_request(droplet_id) + + if droplet_id is not None: + changed = self.delete_droplet_request(droplet_id) + + self.module.exit_json(changed=changed) + +def core(module): + state = module.params['state'] + droplet = DODroplet(module) + if state == 'present': + droplet.create_droplet() + elif state == 'absent': + droplet.delete_droplet() + +def main(): + module = AnsibleModule( + argument_spec=dict( + state=dict(choices=['present', 'absent'], required=True), + api_token = dict( + aliases=['API_TOKEN'], + no_log=True, + fallback=(env_fallback, ['DO_API_TOKEN', 'DO_API_KEY']), + required=True + ), + api_baseurl=dict(type='str', default="https://portal.eclips.is/portal/api/v2"), + name=dict(type='str'), + unique_name=dict(type="bool", default='no'), + size=dict(aliases=['size_id']), + disk=dict(aliases=['disk_size']), + image=dict(type='int', aliases=['image_id']), + region=dict(aliases=['region_id']), + wait=dict(type="bool", default=True), + wait_timeout=dict(type="int", default=60), + ssh_key_id=dict(type='int') + ), + required_together=( + ['size', 'image', 'region', 'disk'], + ), + required_one_of=( + ['id', 'name'], + ), + ) + + try: + core(module) + except DropletException: + e = get_exception() + module.fail_json(msg=e.message) + except RestException: + e = get_exception() + module.fail_json(msg=e.message) + except KeyError: + e = get_exception() + module.fail_json(msg='Unable to load %s' % e.message) + except Exception: + e = get_exception() + module.fail_json(msg=e.message) + +if __name__ == '__main__': + main() diff --git a/deploy/roles/common/tasks/main.yml b/deploy/roles/common/tasks/main.yml new file mode 100644 index 0000000..6cde848 --- /dev/null +++ b/deploy/roles/common/tasks/main.yml @@ -0,0 +1,4 @@ +- name: update the apt cache + raw: apt-get update +- name: bootstrap the system by installing python and python-simplejson + raw: apt-get install -y python python-simplejson diff --git a/deploy/roles/lepidopter_v0.2.1/files/lepidopter-install.sh b/deploy/roles/lepidopter_v0.2.1/files/lepidopter-install.sh new file mode 100644 index 0000000..6edcd4c --- /dev/null +++ b/deploy/roles/lepidopter_v0.2.1/files/lepidopter-install.sh @@ -0,0 +1,96 @@ +#!/bin/sh +# This file was edited also manually. + +export DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true +export LC_ALL=C LANGUAGE=C LANG=C + +# These were added +LEPIDOPTER_REPO_PATH=/opt/lepidopter/ +export DEB_RELEASE="jessie" +export APT_MIRROR="http://httpredir.debian.org/debian" +export ARCH="amd64" +export MIRROR="http://httpredir.debian.org/debian" +export HOSTNAME_IMG="lepidopter" + +set -x +set -e # Exit on first error + +apt-get install -y netbase ntp less openssh-server git-core \ + binutils ca-certificates wget kmod curl \ + haveged lsb-release tcpdump +# XXX disable localepurge as the interactive setup creates problems +# localepurge +#ROOTDIR="$1" + +# Install non-free binary blob and kernel needed to boot Raspberry Pi +# wget https://raw.github.com/Hexxeh/rpi-update/master/rpi-update \ +# -O ${ROOTDIR}/usr/local/sbin/rpi-update +# chmod a+x ${ROOTDIR}/usr/local/sbin/rpi-update +# mkdir -p ${ROOTDIR}/lib/modules +# touch ${ROOTDIR}/boot/start.elf +# Do not try to update itself, avoid making backups, skip warnings +# export UPDATE_SELF=0 SKIP_BACKUP=1 SKIP_WARNING=1 +# chroot ${ROOTDIR} rpi-update + +# Add module for hardware RNG +# cat <> ${ROOTDIR}/etc/modules +# bcm2708_rng +# EOF + +# Add an apt repository with apt preferences +set_apt_sources() { + SUITE="$1" + PIN_PRIORITY="$2" + COMPONENTS="main" + cat <> /etc/apt/sources.list +# Repository: $SUITE +deb $APT_MIRROR $SUITE $COMPONENTS +EOF + if [ -n "$PIN_PRIORITY" ] + then + cat < /etc/apt/preferences.d/${SUITE}.pref +Package: * +Pin: release n=$SUITE +Pin-Priority: $PIN_PRIORITY +EOF + fi +} + +echo "Add Debian ${DEB_RELEASE}-backports repository" +set_apt_sources ${DEB_RELEASE}-backports +echo "Add Debian stretch repository" +set_apt_sources stretch 100 + +# Create ooniprobe log directory +mkdir -p /var/log/ooni/ + +# Copy required scripts, cronjobs and config files to lepidopter +# Rsync Directory/file hieratchy to image +rsync -avp $LEPIDOPTER_REPO_PATH/lepidopter-fh/ / + +# Install ooniprobe via setup script +/setup-ooniprobe.sh +rm /setup-ooniprobe.sh + +# Execute cleanup script +/cleanup.sh +rm /cleanup.sh + +# Remove SSH host keys and add regenerate_ssh_host_keys SYSV script +#/remove_ssh_host_keys.sh +# Don't regenerate ssh keys +rm /etc/init.d/regenerate_ssh_host_keys +rm /remove_ssh_host_keys.sh + +# Remove motd file and create symlink for lepidopter dynamic MOTD +rm /etc/motd +ln -s /var/run/motd /etc/motd + +# Add (optional) pluggable transport support in tor config +cat $LEPIDOPTER_REPO_PATH/conf/tor-pt.conf >> /etc/tor/torrc + +useradd -G sudo lepidopter +echo "lepidopter:lepidopter"|chpasswd + +echo "Customize script finished successfully." +exit 0 diff --git a/deploy/roles/lepidopter_v0.2.1/tasks/main.yml b/deploy/roles/lepidopter_v0.2.1/tasks/main.yml new file mode 100644 index 0000000..bc47111 --- /dev/null +++ b/deploy/roles/lepidopter_v0.2.1/tasks/main.yml @@ -0,0 +1,18 @@ +--- +- name: ensure required packages installed + apt: name={{ item }} state=present + with_items: + - git + - python-pip + - sudo + +- name: clone lepidopter repository + git: repo=https://github.com/TheTorProject/lepidopter.git + dest=/opt/lepidopter + version=v0.2.1-alpha + +- name: run setup script for lepidopter + script: lepidopter-install.sh + +- name: run ooniprobe update to trigger running the updater to the latest version + command: sh /opt/ooni/update-ooniprobe.sh diff --git a/tests/integration/Dockerfile b/tests/integration/Dockerfile new file mode 100644 index 0000000..4449fe6 --- /dev/null +++ b/tests/integration/Dockerfile @@ -0,0 +1,18 @@ +FROM debian:jessie +ARG lepidopter_tag="v0.2.1-alpha" +ARG lepidopter_update_tag="master" + +RUN apt-get update && apt-get install -y \ + git \ + python-pip \ + sudo + +RUN git clone -b $lepidopter_tag https://github.com/TheTorProject/lepidopter.git /opt/lepidopter/ + +RUN mkdir -p /integration +WORKDIR /integration + +RUN git clone -b $lepidopter_update_tag https://github.com/OpenObservatory/lepidopter-update.git /integration/lepidopter-update +RUN lepidopter-update/tests/integration/lepidopter-install-${lepidopter_tag}.sh + +CMD [ "python", "lepidopter-update/tests/integration/main.py" ] diff --git a/tests/integration/lepidopter-install-v0.2.1-alpha.sh b/tests/integration/lepidopter-install-v0.2.1-alpha.sh new file mode 100755 index 0000000..bca70f9 --- /dev/null +++ b/tests/integration/lepidopter-install-v0.2.1-alpha.sh @@ -0,0 +1,101 @@ +#!/bin/sh +# This file was edited also manually. + +export DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true +export LC_ALL=C LANGUAGE=C LANG=C + +# These were added +LEPIDOPTER_REPO_PATH=/opt/lepidopter/ +export DEB_RELEASE="jessie" +export APT_MIRROR="http://httpredir.debian.org/debian" +export ARCH="amd64" +export MIRROR="http://httpredir.debian.org/debian" +export HOSTNAME_IMG="lepidopter" + +set -x +set -e # Exit on first error + +apt-get install -y netbase ntp less openssh-server git-core \ + binutils ca-certificates wget kmod curl \ + haveged lsb-release tcpdump +# XXX disable localepurge as the interactive setup creates problems +# localepurge +#ROOTDIR="$1" + +# Install non-free binary blob and kernel needed to boot Raspberry Pi +# wget https://raw.github.com/Hexxeh/rpi-update/master/rpi-update \ +# -O ${ROOTDIR}/usr/local/sbin/rpi-update +# chmod a+x ${ROOTDIR}/usr/local/sbin/rpi-update +# mkdir -p ${ROOTDIR}/lib/modules +# touch ${ROOTDIR}/boot/start.elf +# Do not try to update itself, avoid making backups, skip warnings +# export UPDATE_SELF=0 SKIP_BACKUP=1 SKIP_WARNING=1 +# chroot ${ROOTDIR} rpi-update + +# Add module for hardware RNG +# cat <> ${ROOTDIR}/etc/modules +# bcm2708_rng +# EOF + +# Add an apt repository with apt preferences +set_apt_sources() { + SUITE="$1" + PIN_PRIORITY="$2" + COMPONENTS="main" + cat <> /etc/apt/sources.list +# Repository: $SUITE +deb $APT_MIRROR $SUITE $COMPONENTS +EOF + if [ -n "$PIN_PRIORITY" ] + then + cat < /etc/apt/preferences.d/${SUITE}.pref +Package: * +Pin: release n=$SUITE +Pin-Priority: $PIN_PRIORITY +EOF + fi +} + +echo "Add Debian ${DEB_RELEASE}-backports repository" +set_apt_sources ${DEB_RELEASE}-backports +echo "Add Debian stretch repository" +set_apt_sources stretch 100 + +# Create ooniprobe log directory +mkdir -p /var/log/ooni/ + +# Copy required scripts, cronjobs and config files to lepidopter +# Rsync Directory/file hieratchy to image +rsync -avp $LEPIDOPTER_REPO_PATH/lepidopter-fh/ / + +# Install the version of ooniprobe in alpha +apt-get -y install openssl libssl-dev libyaml-dev libffi-dev libpcap-dev tor \ + libgeoip-dev libdumbnet-dev python-dev python-pip libgmp-dev +pip install ooniprobe=="1.4.1" + +# Install ooniprobe via setup script +/setup-ooniprobe.sh +rm /setup-ooniprobe.sh + +# Execute cleanup script +/cleanup.sh +rm /cleanup.sh + +# Remove SSH host keys and add regenerate_ssh_host_keys SYSV script +#/remove_ssh_host_keys.sh +# Don't regenerate ssh keys +rm /etc/init.d/regenerate_ssh_host_keys +rm /remove_ssh_host_keys.sh + +# Remove motd file and create symlink for lepidopter dynamic MOTD +rm /etc/motd +ln -s /var/run/motd /etc/motd + +# Add (optional) pluggable transport support in tor config +cat $LEPIDOPTER_REPO_PATH/conf/tor-pt.conf >> /etc/tor/torrc + +useradd -G sudo lepidopter +echo "lepidopter:lepidopter"|chpasswd + +echo "Customize script finished successfully." +exit 0 diff --git a/tests/integration/main.py b/tests/integration/main.py new file mode 100644 index 0000000..34b4f13 --- /dev/null +++ b/tests/integration/main.py @@ -0,0 +1,91 @@ +import os +import re +import imp +import time +import signal +import operator +import subprocess +import multiprocessing + +import unittest + +UPDATERS_DIR = os.path.abspath( + os.path.join( + os.path.dirname(__file__), + '..', '..', + 'updater', + 'versions' + ) +) + +UPDATER_FILE_REGEXP = re.compile(r"^update-(\d).py$") + +def kill_children(pid, sig=signal.SIGTERM): + p = subprocess.Popen("ps -o pid --ppid %d --noheaders" % pid, + shell=True, stdout=subprocess.PIPE) + output = p.stdout.read() + p.wait() + for child_pid in output.split("\n")[:-1]: + os.kill(int(child_pid), sig) + +class TestUpdaters(unittest.TestCase): + def setUp(self): + if not os.path.exists("/etc/default/lepidopter"): + raise RuntimeError("These must be run on a system that looks like lepidopter") + + def list_updaters(self): + updaters = [] + for filename in os.listdir(UPDATERS_DIR): + m = UPDATER_FILE_REGEXP.match(filename) + if m: + version = int(m.group(1)) + path = os.path.join(UPDATERS_DIR, filename) + module = imp.load_source( + 'updater_{0}'.format(version), + path + ) + updaters.append({ + 'version': version, + 'module': module, + 'path': path + }) + updaters.sort(key=operator.itemgetter('version')) + return updaters + + def are_versions_consistent(self, updaters): + last_version = 0 + for updater in updaters: + # Ensure versions versions match the filename + self.assertEqual(updater['module'].__version__, + str(updater['version']), + 'Mismatch in __version__ variable of for updater-%s.py' % updater['version']) + # Ensure versions always increase by one + self.assertEqual(last_version + 1, updater['version'], + 'Missing version %s' % (last_version + 1)) + last_version = updater['version'] + + def updaters_can_be_partial(self, updaters): + for updater in updaters: + p = multiprocessing.Process(target=updater['module'].run) + p.start() + time.sleep(0.5) + kill_children(p.pid) + p.terminate() + + p = multiprocessing.Process(target=updater['module'].run) + p.start() + p.join() + self.assertEqual(p.exitcode, 0) + + def updaters_smoke_test(self, updaters): + for updater in updaters: + updater['module'].run() + + def test_update_lifecycle(self): + updaters = self.list_updaters() + self.are_versions_consistent(updaters) + self.updaters_can_be_partial(updaters) + self.updaters_smoke_test(updaters) + +if __name__ == "__main__": + unittest.main() diff --git a/tests/integration/setup.sh b/tests/integration/setup.sh new file mode 100644 index 0000000..1875a9d --- /dev/null +++ b/tests/integration/setup.sh @@ -0,0 +1,5 @@ +#!/bin/sh +mkdir -p /etc/default +cat << EOF > /etc/default/lepidopter +LEPIDOPTER_BUILD="alpha" +EOF diff --git a/updater/test_updater.py b/updater/test_updater.py index 7294ff2..1d1e613 100644 --- a/updater/test_updater.py +++ b/updater/test_updater.py @@ -9,6 +9,7 @@ class TestSecurity(unittest.TestCase): def test_verify_file(self): o = verify_file( os.path.join(CWD, "versions", "update-1.py.asc"), + os.path.join(CWD, "versions", "update-1.py"), os.path.join(CWD, "public.asc") ) print o