From ce668bf34ab96057d38ea233d5b84a415492ffda Mon Sep 17 00:00:00 2001
From: Fitz Elliott <felliott@fiskur.org>
Date: Thu, 27 Feb 2014 14:54:25 -0500
Subject: [PATCH 01/23] add elasticsearch daemon task

---
 tasks.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tasks.py b/tasks.py
index 82cfb39..4c5379b 100644
--- a/tasks.py
+++ b/tasks.py
@@ -14,6 +14,13 @@ def mongo(daemon=False, port=20771):
         cmd += " --fork"
     run(cmd)
 
+@task
+def elastic(port=20772, clustername='modular-odm-test'):
+    '''Run the elasticsearch process.
+    '''
+    cmd = "elasticsearch -Des.http.port={0} -Des.cluster.name={1}".format(port, clustername)
+    run(cmd)
+
 @task
 def test(coverage=False, browse=False):
     command = "nosetests"

From e1ce884d7e830dac618c1aeaea02cf4d0820889d Mon Sep 17 00:00:00 2001
From: Fitz Elliott <felliott@fiskur.org>
Date: Sun, 2 Mar 2014 10:14:49 -0500
Subject: [PATCH 02/23] first pass at new elasticsearchstorage.py

 * This is essentially just a search and replace on mongostorage.py
---
 modularodm/storage/__init__.py             |   1 +
 modularodm/storage/elasticsearchstorage.py | 195 +++++++++++++++++++++
 requirements.txt                           |   1 +
 3 files changed, 197 insertions(+)
 create mode 100644 modularodm/storage/elasticsearchstorage.py

diff --git a/modularodm/storage/__init__.py b/modularodm/storage/__init__.py
index 4f2b46c..17cb2ef 100644
--- a/modularodm/storage/__init__.py
+++ b/modularodm/storage/__init__.py
@@ -2,3 +2,4 @@
 from .mongostorage import MongoStorage
 from .picklestorage import PickleStorage
 from .ephemeralstorage import EphemeralStorage
+from .elasticsearchstorage import ElasticsearchStorage
diff --git a/modularodm/storage/elasticsearchstorage.py b/modularodm/storage/elasticsearchstorage.py
new file mode 100644
index 0000000..69a645e
--- /dev/null
+++ b/modularodm/storage/elasticsearchstorage.py
@@ -0,0 +1,195 @@
+import elasticsearch
+
+from .base import Storage
+from ..query.queryset import BaseQuerySet
+from ..query.query import QueryGroup
+from ..query.query import RawQuery
+from modularodm.exceptions import NoResultsFound, MultipleResultsFound
+
+
+class ElasticsearchQuerySet(BaseQuerySet):
+
+    def __init__(self, schema, cursor):
+
+        super(ElasticsearchQuerySet, self).__init__(schema)
+        self.data = cursor
+
+    def __getitem__(self, index, raw=False):
+        super(ElasticsearchQuerySet, self).__getitem__(index)
+        key = self.data[index][self.primary]
+        if raw:
+            return key
+        return self.schema.load(key)
+
+    def __iter__(self, raw=False):
+        keys = [obj[self.primary] for obj in self.data.clone()]
+        if raw:
+            return keys
+        return (self.schema.load(key) for key in keys)
+
+    def __len__(self):
+
+        return self.data.count(with_limit_and_skip=True)
+
+    count = __len__
+
+    def get_key(self, index):
+        return self.__getitem__(index, raw=True)
+
+    def get_keys(self):
+        return list(self.__iter__(raw=True))
+
+    def sort(self, *keys):
+
+        sort_key = []
+
+        for key in keys:
+
+            if key.startswith('-'):
+                key = key.lstrip('-')
+                sign = pyelasticsearch.DESCENDING
+            else:
+                sign = pyelasticsearch.ASCENDING
+
+            sort_key.append((key, sign))
+
+        self.data = self.data.sort(sort_key)
+        return self
+
+    def offset(self, n):
+
+        self.data = self.data.skip(n)
+        return self
+
+    def limit(self, n):
+
+        self.data = self.data.limit(n)
+        return self
+
+class ElasticsearchStorage(Storage):
+
+    QuerySet = ElasticsearchQuerySet
+
+    def __init__(self, client, collection):
+        self.client = client
+        self.collection = collection
+
+    def find(self, query=None, **kwargs):
+        elasticsearch_query = self._translate_query(query)
+        return self.client.search(
+            index=self.collection,
+            body=elasticsearch_query,
+        )
+
+    def find_one(self, query=None, **kwargs):
+        """ Gets a single object from the collection.
+
+        If no matching documents are found, raises ``NoResultsFound``.
+        If >1 matching documents are found, raises ``MultipleResultsFound``.
+
+        :params: One or more ``Query`` or ``QuerySet`` objects may be passed
+
+        :returns: The selected document
+        """
+        elasticsearch_query = self._translate_query(query)
+        matches = self.client.search(
+            index=self.collection,
+            body=elasticsearch_query,
+        )
+
+        if matches.count() == 1:
+            return matches[0]
+
+        if matches.count() == 0:
+            raise NoResultsFound()
+
+        raise MultipleResultsFound(
+            'Query for find_one must return exactly one result; '
+            'returned {0}'.format(matches.count())
+        )
+
+    def get(self, primary_name, key):
+        return self.client.get(index=self.collection, doc_type=primary_name, id=key)
+
+    def insert(self, primary_name, key, value):
+        if primary_name not in value:
+            value = value.copy()
+            value[primary_name] = key
+        self.client.create(index=self.collection, doc_type=primary_name, id=key, body=value)
+
+    def update(self, query, data):
+
+        elasticsearch_query = self._translate_query(query)
+        if '_id' in elasticsearch_query:
+            update_data = {k: v for k, v in data.items() if k != '_id'}
+        else:
+            update_data = data
+        update_query = {'$set': update_data}
+
+        self.client.update(
+            index=self.collection,
+            doc_type=primary_name, id=key,
+            body=data
+        )
+
+    def remove(self, query=None):
+        elasticsearch_query = self._translate_query(query)
+        self.client.delete_by_query(index=self.collection, body=elasticsearch_query)
+
+    def flush(self):
+        pass
+
+    def __repr__(self):
+        return self.find()
+
+    def _translate_query(self, query=None, elasticsearch_query=None):
+        """
+
+        """
+        elasticsearch_query = elasticsearch_query or {}
+
+        if isinstance(query, RawQuery):
+            attribute, operator, argument = \
+                query.attribute, query.operator, query.argument
+
+            if operator == 'eq':
+                elasticsearch_query[attribute] = argument
+
+            elif operator in COMPARISON_OPERATORS:
+                elasticsearch_operator = '$' + operator
+                if attribute not in elasticsearch_query:
+                    elasticsearch_query[attribute] = {}
+                elasticsearch_query[attribute][elasticsearch_operator] = argument
+
+            elif operator in STRING_OPERATORS:
+                elasticsearch_operator = '$regex'
+                elasticsearch_regex = prepare_query_value(operator, argument)
+                if attribute not in elasticsearch_query:
+                    elasticsearch_query[attribute] = {}
+                elasticsearch_query[attribute][elasticsearch_operator] = elasticsearch_regex
+
+        elif isinstance(query, QueryGroup):
+
+            if query.operator == 'and':
+                elasticsearch_query = {}
+                for node in query.nodes:
+                    part = self._translate_query(node, elasticsearch_query)
+                    elasticsearch_query.update(part)
+                return elasticsearch_query
+
+            elif query.operator == 'or':
+                return {'$or' : [self._translate_query(node) for node in query.nodes]}
+
+            elif query.operator == 'not':
+                return {'$not' : self._translate_query(query.nodes[0])}
+
+            else:
+                raise ValueError('QueryGroup operator must be <and>, <or>, or <not>.')
+
+        elif query is None:
+            return {}
+
+        else:
+            raise TypeError('Query must be a QueryGroup or Query object.')
+
+        return elasticsearch_query
diff --git a/requirements.txt b/requirements.txt
index 784fc18..12f36d6 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,3 +2,4 @@ flask
 blinker
 pymongo
 python-dateutil
+elasticsearch

From 031cdc88925a9a0c0469b9634d8fa868185cbc7b Mon Sep 17 00:00:00 2001
From: Fitz Elliott <felliott@fiskur.org>
Date: Sun, 2 Mar 2014 10:16:03 -0500
Subject: [PATCH 03/23] add Elasticsearch test object

---
 tests/base.py | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/tests/base.py b/tests/base.py
index b143ad8..220a951 100644
--- a/tests/base.py
+++ b/tests/base.py
@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+import elasticsearch
 import logging
 import inspect
 import os
@@ -7,7 +8,7 @@
 import uuid
 
 from modularodm import StoredObject
-from modularodm.storage import MongoStorage, PickleStorage, EphemeralStorage
+from modularodm.storage import MongoStorage, PickleStorage, EphemeralStorage, ElasticsearchStorage
 
 logger = logging.getLogger(__name__)
 
@@ -83,6 +84,23 @@ def clean_up_storage(self):
             self.mongo_client.drop_collection(c)
 
 
+class ElasticsearchStorageMixin(object):
+    fixture_suffix = 'Elasticsearch'
+
+    # DB settings
+    DB_HOST = os.environ.get('ES_HOST', 'localhost')
+    DB_PORT = int(os.environ.get('ES_PORT', '20772'))
+
+    client = elasticsearch.Elasticsearch([{"host": DB_HOST, "port": DB_PORT},])
+
+    def make_storage(self):
+        collection = str(uuid.uuid4())[:8]
+        return ElasticsearchStorage(client=self.client, collection=collection)
+
+    def clean_up_storage(self):
+        self.client.flush()
+
+
 class MultipleBackendMeta(type):
     def __new__(mcs, name, bases, dct):
 
@@ -101,6 +119,7 @@ def __new__(mcs, name, bases, dct):
             PickleStorageMixin,
             MongoStorageMixin,
             EphemeralStorageMixin,
+            ElasticsearchStorageMixin,
         ):
             new_name = '{}{}'.format(name, mixin.fixture_suffix)
             frame.f_globals[new_name] = type.__new__(

From 56dc7ecf3405cd775441614d8729ae88fa9dc493 Mon Sep 17 00:00:00 2001
From: Fitz Elliott <felliott@fiskur.org>
Date: Sun, 2 Mar 2014 10:47:26 -0500
Subject: [PATCH 04/23] collection != index: user must provide index to ES

 * I assumed that a modular-odm collection was the same thing as an
   elasticsearch index.  Nope. A collection is the doc-type in ES
   parlance.  Add a required es_index attribute to ElasticsearchStorage
   constructor and update ES calls to use correct nomenclature.

 * The test ES storage object now provides es_index.
---
 modularodm/storage/elasticsearchstorage.py | 21 ++++++++++-----------
 tests/base.py                              |  4 +++-
 2 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/modularodm/storage/elasticsearchstorage.py b/modularodm/storage/elasticsearchstorage.py
index 69a645e..a0b04e4 100644
--- a/modularodm/storage/elasticsearchstorage.py
+++ b/modularodm/storage/elasticsearchstorage.py
@@ -70,14 +70,16 @@ class ElasticsearchStorage(Storage):
 
     QuerySet = ElasticsearchQuerySet
 
-    def __init__(self, client, collection):
+    def __init__(self, client, es_index, collection, ):
         self.client = client
         self.collection = collection
+        self.es_index = es_index
+
 
     def find(self, query=None, **kwargs):
         elasticsearch_query = self._translate_query(query)
         return self.client.search(
-            index=self.collection,
+            index=self.es_index,
             body=elasticsearch_query,
         )
 
@@ -93,7 +95,7 @@ def find_one(self, query=None, **kwargs):
         """
         elasticsearch_query = self._translate_query(query)
         matches = self.client.search(
-            index=self.collection,
+            index=self.es_index,
             body=elasticsearch_query,
         )
 
@@ -109,13 +111,10 @@ def find_one(self, query=None, **kwargs):
         )
 
     def get(self, primary_name, key):
-        return self.client.get(index=self.collection, doc_type=primary_name, id=key)
+        return self.client.get(index=self.es_index, doc_type=self.collection, id=key)
 
     def insert(self, primary_name, key, value):
-        if primary_name not in value:
-            value = value.copy()
-            value[primary_name] = key
-        self.client.create(index=self.collection, doc_type=primary_name, id=key, body=value)
+        self.client.create(index=self.es_index, doc_type=self.collection, id=key, body=value)
 
     def update(self, query, data):
 
@@ -127,14 +126,14 @@ def update(self, query, data):
         update_query = {'$set': update_data}
 
         self.client.update(
-            index=self.collection,
-            doc_type=primary_name, id=key,
+            index=self.es_index,
+            doc_type=self.collection, id=key,
             body=data
         )
 
     def remove(self, query=None):
         elasticsearch_query = self._translate_query(query)
-        self.client.delete_by_query(index=self.collection, body=elasticsearch_query)
+        self.client.delete_by_query(index=self.es_index, body=elasticsearch_query)
 
     def flush(self):
         pass
diff --git a/tests/base.py b/tests/base.py
index 220a951..8015770 100644
--- a/tests/base.py
+++ b/tests/base.py
@@ -95,7 +95,9 @@ class ElasticsearchStorageMixin(object):
 
     def make_storage(self):
         collection = str(uuid.uuid4())[:8]
-        return ElasticsearchStorage(client=self.client, collection=collection)
+        return ElasticsearchStorage(
+            client=self.client, es_index="--modmtest--", collection=collection
+        )
 
     def clean_up_storage(self):
         self.client.flush()

From 5e2f3e5c71db131ddf108844f8ebfa0bd9bee43d Mon Sep 17 00:00:00 2001
From: Fitz Elliott <felliott@fiskur.org>
Date: Sun, 2 Mar 2014 10:52:00 -0500
Subject: [PATCH 05/23] fix crash in ElasticsearchQuerySet

 * ElasticsearchQuerySet.data is an array, not a real cursor. Stop
   calling count() on it, just use len() instead.
---
 modularodm/storage/elasticsearchstorage.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/modularodm/storage/elasticsearchstorage.py b/modularodm/storage/elasticsearchstorage.py
index a0b04e4..37fb9f6 100644
--- a/modularodm/storage/elasticsearchstorage.py
+++ b/modularodm/storage/elasticsearchstorage.py
@@ -28,8 +28,7 @@ def __iter__(self, raw=False):
         return (self.schema.load(key) for key in keys)
 
     def __len__(self):
-
-        return self.data.count(with_limit_and_skip=True)
+        return len(self.data)
 
     count = __len__
 

From 2f3968eec3e67e2b843feb8ed2ba9a646f582864 Mon Sep 17 00:00:00 2001
From: Fitz Elliott <felliott@fiskur.org>
Date: Sun, 2 Mar 2014 12:04:38 -0500
Subject: [PATCH 06/23] delete ES test index when done

---
 tests/base.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/base.py b/tests/base.py
index 8015770..2b8cbfb 100644
--- a/tests/base.py
+++ b/tests/base.py
@@ -90,17 +90,20 @@ class ElasticsearchStorageMixin(object):
     # DB settings
     DB_HOST = os.environ.get('ES_HOST', 'localhost')
     DB_PORT = int(os.environ.get('ES_PORT', '20772'))
+    DB_INDEX = '--modmtest--'
 
     client = elasticsearch.Elasticsearch([{"host": DB_HOST, "port": DB_PORT},])
+    indices = elasticsearch.client.IndicesClient(client)
 
     def make_storage(self):
         collection = str(uuid.uuid4())[:8]
         return ElasticsearchStorage(
-            client=self.client, es_index="--modmtest--", collection=collection
+            client=self.client, es_index=self.DB_INDEX, collection=collection
         )
 
     def clean_up_storage(self):
         self.client.flush()
+        self.indices.delete(self.DB_INDEX)
 
 
 class MultipleBackendMeta(type):

From 44b53b099f5c57ebb1f01555200804f0b498be48 Mon Sep 17 00:00:00 2001
From: Fitz Elliott <felliott@fiskur.org>
Date: Sun, 2 Mar 2014 12:09:43 -0500
Subject: [PATCH 07/23] refresh ES index after inserting fixtures

 * ES does real-time search, so we have to refresh the index to make
   sure all of the fixtures have been inserted before searching.
---
 tests/base.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/base.py b/tests/base.py
index 2b8cbfb..51ef8e2 100644
--- a/tests/base.py
+++ b/tests/base.py
@@ -105,6 +105,10 @@ def clean_up_storage(self):
         self.client.flush()
         self.indices.delete(self.DB_INDEX)
 
+    def setUp(self):
+        super(ElasticsearchStorageMixin, self).setUp()
+        self.indices.refresh(index=self.DB_INDEX)
+
 
 class MultipleBackendMeta(type):
     def __new__(mcs, name, bases, dct):

From 51c7e16a12bd06d7231e1dbac74affc3094fb5f3 Mon Sep 17 00:00:00 2001
From: Fitz Elliott <felliott@fiskur.org>
Date: Sun, 2 Mar 2014 12:17:49 -0500
Subject: [PATCH 08/23] fill out storage retrieval methods

 * Get find(), find_one(), update(), remove() working properly
---
 modularodm/storage/elasticsearchstorage.py | 48 ++++++++++++----------
 1 file changed, 27 insertions(+), 21 deletions(-)

diff --git a/modularodm/storage/elasticsearchstorage.py b/modularodm/storage/elasticsearchstorage.py
index 37fb9f6..3f083df 100644
--- a/modularodm/storage/elasticsearchstorage.py
+++ b/modularodm/storage/elasticsearchstorage.py
@@ -1,4 +1,4 @@
-import elasticsearch
+from elasticsearch import helpers
 
 from .base import Storage
 from ..query.queryset import BaseQuerySet
@@ -77,10 +77,18 @@ def __init__(self, client, es_index, collection, ):
 
     def find(self, query=None, **kwargs):
         elasticsearch_query = self._translate_query(query)
-        return self.client.search(
+
+        matches = []
+        for results in helpers.scan(
+            self.client,
+            query=elasticsearch_query,
             index=self.es_index,
-            body=elasticsearch_query,
-        )
+            doc_type=self.collection,
+        ):
+            matches.append(results)
+
+
+        return matches
 
     def find_one(self, query=None, **kwargs):
         """ Gets a single object from the collection.
@@ -95,18 +103,19 @@ def find_one(self, query=None, **kwargs):
         elasticsearch_query = self._translate_query(query)
         matches = self.client.search(
             index=self.es_index,
+            doc_type=self.collection,
             body=elasticsearch_query,
-        )
+        )['hits']['hits']
 
-        if matches.count() == 1:
+        if len(matches) == 1:
             return matches[0]
 
-        if matches.count() == 0:
+        if len(matches) == 0:
             raise NoResultsFound()
 
         raise MultipleResultsFound(
             'Query for find_one must return exactly one result; '
-            'returned {0}'.format(matches.count())
+            'returned {0}'.format(len(matches))
         )
 
     def get(self, primary_name, key):
@@ -116,24 +125,21 @@ def insert(self, primary_name, key, value):
         self.client.create(index=self.es_index, doc_type=self.collection, id=key, body=value)
 
     def update(self, query, data):
+        for doc in self.find(query):
+            self.client.update(
+                index=self.es_index,
+                doc_type=self.collection, id=doc['_id'],
+                body={'doc': data}
+            )
 
+    def remove(self, query=None):
         elasticsearch_query = self._translate_query(query)
-        if '_id' in elasticsearch_query:
-            update_data = {k: v for k, v in data.items() if k != '_id'}
-        else:
-            update_data = data
-        update_query = {'$set': update_data}
-
-        self.client.update(
+        self.client.delete_by_query(
             index=self.es_index,
-            doc_type=self.collection, id=key,
-            body=data
+            doc_type=self.collection,
+            body=elasticsearch_query
         )
 
-    def remove(self, query=None):
-        elasticsearch_query = self._translate_query(query)
-        self.client.delete_by_query(index=self.es_index, body=elasticsearch_query)
-
     def flush(self):
         pass
 

From 6932b08b93dc0e2727ac1175d9398fbb4512fcca Mon Sep 17 00:00:00 2001
From: Fitz Elliott <felliott@fiskur.org>
Date: Sun, 2 Mar 2014 12:20:38 -0500
Subject: [PATCH 09/23] add search by query functionality

 * Turn modular-odm query objects into filter structures suitable for
   passing to Elasticsearch. We now support everything in the tests
   except for icontains.
---
 modularodm/storage/elasticsearchstorage.py | 62 +++++++++++++---------
 1 file changed, 38 insertions(+), 24 deletions(-)

diff --git a/modularodm/storage/elasticsearchstorage.py b/modularodm/storage/elasticsearchstorage.py
index 3f083df..40b85c9 100644
--- a/modularodm/storage/elasticsearchstorage.py
+++ b/modularodm/storage/elasticsearchstorage.py
@@ -6,13 +6,20 @@
 from ..query.query import RawQuery
 from modularodm.exceptions import NoResultsFound, MultipleResultsFound
 
+EQUALITY_OPERATORS = ('eq', 'ne')
+SET_OPERATORS = ('in', 'nin')
+RANGE_OPERATORS = ('gt', 'gte', 'lt', 'lte')
+NEGATION_OPERATORS = ('ne', 'nin')
+
+STRING_OPERATORS = ('contains', 'icontains', 'endswith')
+STRINGOP_MAP = {'contains': '.*%s.*', 'icontains': '.*%s.*', 'endswith': '.*%s',}
 
 class ElasticsearchQuerySet(BaseQuerySet):
 
-    def __init__(self, schema, cursor):
+    def __init__(self, schema, data):
 
         super(ElasticsearchQuerySet, self).__init__(schema)
-        self.data = cursor
+        self.data = list(data)
 
     def __getitem__(self, index, raw=False):
         super(ElasticsearchQuerySet, self).__getitem__(index)
@@ -146,46 +153,49 @@ def flush(self):
     def __repr__(self):
         return self.find()
 
+
     def _translate_query(self, query=None, elasticsearch_query=None):
-        """
+        elasticsearch_query = self._build_query(query, elasticsearch_query)
+        return {'filter' : elasticsearch_query}
 
-        """
+
+    def _build_query(self, query=None, elasticsearch_query=None):
         elasticsearch_query = elasticsearch_query or {}
 
         if isinstance(query, RawQuery):
             attribute, operator, argument = \
                 query.attribute, query.operator, query.argument
 
-            if operator == 'eq':
-                elasticsearch_query[attribute] = argument
+            if operator in EQUALITY_OPERATORS:
+                elasticsearch_query['term'] = {attribute: argument}
+
+            elif operator in RANGE_OPERATORS:
+                elasticsearch_query['range'] = {attribute: {operator: argument}}
 
-            elif operator in COMPARISON_OPERATORS:
-                elasticsearch_operator = '$' + operator
-                if attribute not in elasticsearch_query:
-                    elasticsearch_query[attribute] = {}
-                elasticsearch_query[attribute][elasticsearch_operator] = argument
+            elif operator in SET_OPERATORS:
+                elasticsearch_query['terms'] = {attribute: argument}
+
+            elif operator == 'startswith':
+                elasticsearch_query['prefix'] = {attribute: argument}
 
             elif operator in STRING_OPERATORS:
-                elasticsearch_operator = '$regex'
-                elasticsearch_regex = prepare_query_value(operator, argument)
-                if attribute not in elasticsearch_query:
-                    elasticsearch_query[attribute] = {}
-                elasticsearch_query[attribute][elasticsearch_operator] = elasticsearch_regex
+                elasticsearch_query['regexp'] = {
+                    attribute: self._stringop_to_regex(operator, argument)
+                }
 
-        elif isinstance(query, QueryGroup):
 
+            if operator in NEGATION_OPERATORS:
+                elasticsearch_query = {"not": elasticsearch_query}
+
+        elif isinstance(query, QueryGroup):
             if query.operator == 'and':
-                elasticsearch_query = {}
-                for node in query.nodes:
-                    part = self._translate_query(node, elasticsearch_query)
-                    elasticsearch_query.update(part)
-                return elasticsearch_query
+                return {'and' : [self._build_query(node) for node in query.nodes]}
 
             elif query.operator == 'or':
-                return {'$or' : [self._translate_query(node) for node in query.nodes]}
+                return {'or' : [self._build_query(node) for node in query.nodes]}
 
             elif query.operator == 'not':
-                return {'$not' : self._translate_query(query.nodes[0])}
+                return {'not' : self._build_query(query.nodes[0])}
 
             else:
                 raise ValueError('QueryGroup operator must be <and>, <or>, or <not>.')
@@ -197,3 +207,7 @@ def _translate_query(self, query=None, elasticsearch_query=None):
             raise TypeError('Query must be a QueryGroup or Query object.')
 
         return elasticsearch_query
+
+    def _stringop_to_regex(self, operator, argument):
+        return STRINGOP_MAP[operator] % argument
+

From dc0fec8fc8ffe6217cb7e27ad4689dffe7d95f2d Mon Sep 17 00:00:00 2001
From: Fitz Elliott <felliott@fiskur.org>
Date: Sun, 2 Mar 2014 12:22:24 -0500
Subject: [PATCH 10/23] EsQuerySet now copies PickleQuerySet

 * Since we're storing a list of results rather than an actual cursor,
   our queryset implementation is basically the same as Pickle's rather
   than Mongo's.
---
 modularodm/storage/elasticsearchstorage.py | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/modularodm/storage/elasticsearchstorage.py b/modularodm/storage/elasticsearchstorage.py
index 40b85c9..fed0352 100644
--- a/modularodm/storage/elasticsearchstorage.py
+++ b/modularodm/storage/elasticsearchstorage.py
@@ -29,7 +29,7 @@ def __getitem__(self, index, raw=False):
         return self.schema.load(key)
 
     def __iter__(self, raw=False):
-        keys = [obj[self.primary] for obj in self.data.clone()]
+        keys = [obj[self.primary] for obj in self.data]
         if raw:
             return keys
         return (self.schema.load(key) for key in keys)
@@ -48,28 +48,26 @@ def get_keys(self):
     def sort(self, *keys):
 
         sort_key = []
-
-        for key in keys:
+        for key in keys[::-1]:
 
             if key.startswith('-'):
+                reverse = True
                 key = key.lstrip('-')
-                sign = pyelasticsearch.DESCENDING
             else:
-                sign = pyelasticsearch.ASCENDING
+                reverse = False
 
-            sort_key.append((key, sign))
+            self.data = sorted(self.data, key=lambda record: record[key], reverse=reverse)
 
-        self.data = self.data.sort(sort_key)
         return self
 
     def offset(self, n):
 
-        self.data = self.data.skip(n)
+        self.data = self.data[n:]
         return self
 
     def limit(self, n):
 
-        self.data = self.data.limit(n)
+        self.data = self.data[:n]
         return self
 
 class ElasticsearchStorage(Storage):

From 5b221ce92b72bfb861dcfc115ece52b098e718da Mon Sep 17 00:00:00 2001
From: Fitz Elliott <felliott@fiskur.org>
Date: Sun, 2 Mar 2014 12:24:15 -0500
Subject: [PATCH 11/23] add stub for converting ES resp. to native types

 * ES is returning integer ID fields as strings instead of integers. Add
   a stub _to_native_types method where casting will take place.
---
 modularodm/storage/elasticsearchstorage.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/modularodm/storage/elasticsearchstorage.py b/modularodm/storage/elasticsearchstorage.py
index fed0352..a276924 100644
--- a/modularodm/storage/elasticsearchstorage.py
+++ b/modularodm/storage/elasticsearchstorage.py
@@ -92,6 +92,8 @@ def find(self, query=None, **kwargs):
         ):
             matches.append(results)
 
+        for match in matches:
+            self._to_native_types(match)
 
         return matches
 
@@ -112,6 +114,9 @@ def find_one(self, query=None, **kwargs):
             body=elasticsearch_query,
         )['hits']['hits']
 
+        for match in matches:
+            self._to_native_types(match)
+
         if len(matches) == 1:
             return matches[0]
 
@@ -124,7 +129,9 @@ def find_one(self, query=None, **kwargs):
         )
 
     def get(self, primary_name, key):
-        return self.client.get(index=self.es_index, doc_type=self.collection, id=key)
+        match = self.client.get(index=self.es_index, doc_type=self.collection, id=key)
+        self._to_native_types(match)
+        return match
 
     def insert(self, primary_name, key, value):
         self.client.create(index=self.es_index, doc_type=self.collection, id=key, body=value)
@@ -209,3 +216,5 @@ def _build_query(self, query=None, elasticsearch_query=None):
     def _stringop_to_regex(self, operator, argument):
         return STRINGOP_MAP[operator] % argument
 
+    def _to_native_types(self, match):
+        return match

From fdb87ead25c423e247ecbdc80b7f6226c4137fd1 Mon Sep 17 00:00:00 2001
From: Fitz Elliott <felliott@fiskur.org>
Date: Sun, 2 Mar 2014 19:05:04 -0500
Subject: [PATCH 12/23] doc/style cleanup

---
 modularodm/storage/elasticsearchstorage.py | 11 ++++-------
 tests/base.py                              |  3 ++-
 2 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/modularodm/storage/elasticsearchstorage.py b/modularodm/storage/elasticsearchstorage.py
index a276924..30a94ee 100644
--- a/modularodm/storage/elasticsearchstorage.py
+++ b/modularodm/storage/elasticsearchstorage.py
@@ -17,7 +17,6 @@
 class ElasticsearchQuerySet(BaseQuerySet):
 
     def __init__(self, schema, data):
-
         super(ElasticsearchQuerySet, self).__init__(schema)
         self.data = list(data)
 
@@ -46,7 +45,6 @@ def get_keys(self):
         return list(self.__iter__(raw=True))
 
     def sort(self, *keys):
-
         sort_key = []
         for key in keys[::-1]:
 
@@ -61,12 +59,10 @@ def sort(self, *keys):
         return self
 
     def offset(self, n):
-
         self.data = self.data[n:]
         return self
 
     def limit(self, n):
-
         self.data = self.data[:n]
         return self
 
@@ -79,10 +75,12 @@ def __init__(self, client, es_index, collection, ):
         self.collection = collection
         self.es_index = es_index
 
-
     def find(self, query=None, **kwargs):
         elasticsearch_query = self._translate_query(query)
 
+        # elasticsearch *always* limits search results.  The scan()
+        # helper in the elasitcsearch package will make repeated
+        # requests until no more results are returned.
         matches = []
         for results in helpers.scan(
             self.client,
@@ -158,13 +156,12 @@ def flush(self):
     def __repr__(self):
         return self.find()
 
-
     def _translate_query(self, query=None, elasticsearch_query=None):
         elasticsearch_query = self._build_query(query, elasticsearch_query)
         return {'filter' : elasticsearch_query}
 
-
     def _build_query(self, query=None, elasticsearch_query=None):
+        """Turn a query object into a valid elasticsearch filter dict"""
         elasticsearch_query = elasticsearch_query or {}
 
         if isinstance(query, RawQuery):
diff --git a/tests/base.py b/tests/base.py
index 51ef8e2..0ca1683 100644
--- a/tests/base.py
+++ b/tests/base.py
@@ -92,7 +92,7 @@ class ElasticsearchStorageMixin(object):
     DB_PORT = int(os.environ.get('ES_PORT', '20772'))
     DB_INDEX = '--modmtest--'
 
-    client = elasticsearch.Elasticsearch([{"host": DB_HOST, "port": DB_PORT},])
+    client = elasticsearch.Elasticsearch([{"host": DB_HOST, "port": DB_PORT}])
     indices = elasticsearch.client.IndicesClient(client)
 
     def make_storage(self):
@@ -105,6 +105,7 @@ def clean_up_storage(self):
         self.client.flush()
         self.indices.delete(self.DB_INDEX)
 
+    # must refresh index before querying, or some results may not be found
     def setUp(self):
         super(ElasticsearchStorageMixin, self).setUp()
         self.indices.refresh(index=self.DB_INDEX)

From 633a2eaed3f67c3bd02dc076fd5798f3d39e27c2 Mon Sep 17 00:00:00 2001
From: Fitz Elliott <felliott@fiskur.org>
Date: Mon, 10 Mar 2014 15:22:49 -0400
Subject: [PATCH 13/23] fix ES delete query syntax

 * delete_by_query() won't accept a plain filter as the query
   body. Instead, pass a "filtered" query that does a match_all + filter
---
 modularodm/storage/elasticsearchstorage.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/modularodm/storage/elasticsearchstorage.py b/modularodm/storage/elasticsearchstorage.py
index 30a94ee..d3335d5 100644
--- a/modularodm/storage/elasticsearchstorage.py
+++ b/modularodm/storage/elasticsearchstorage.py
@@ -144,10 +144,14 @@ def update(self, query, data):
 
     def remove(self, query=None):
         elasticsearch_query = self._translate_query(query)
+        delete_query = {"filtered" : {
+            "query" : {"match_all" : {}},
+            "filter": elasticsearch_query['filter'],
+        }}
         self.client.delete_by_query(
             index=self.es_index,
             doc_type=self.collection,
-            body=elasticsearch_query
+            body=delete_query
         )
 
     def flush(self):

From 0cf5fa8d9d83a22aec4534bad3a09c40b644881e Mon Sep 17 00:00:00 2001
From: Fitz Elliott <felliott@fiskur.org>
Date: Tue, 11 Mar 2014 08:12:03 -0400
Subject: [PATCH 14/23] add hacky workaround to get backrefs working

 * Backrefs are stored as tuples of (id, ref_name).  If id is an
   integer, then Elasticsearch assumes all elements in the tuple will be
   integers and chokes when it encounters ref_name (a string).  For now,
   explicitly cast the first element of a tuple as a str().
---
 modularodm/storage/elasticsearchstorage.py | 39 ++++++++++++++++------
 1 file changed, 28 insertions(+), 11 deletions(-)

diff --git a/modularodm/storage/elasticsearchstorage.py b/modularodm/storage/elasticsearchstorage.py
index d3335d5..e576730 100644
--- a/modularodm/storage/elasticsearchstorage.py
+++ b/modularodm/storage/elasticsearchstorage.py
@@ -1,4 +1,4 @@
-from elasticsearch import helpers
+from elasticsearch import helpers, NotFoundError
 
 from .base import Storage
 from ..query.queryset import BaseQuerySet
@@ -91,7 +91,7 @@ def find(self, query=None, **kwargs):
             matches.append(results)
 
         for match in matches:
-            self._to_native_types(match)
+            match = self._from_elastic_types(match)
 
         return matches
 
@@ -112,11 +112,8 @@ def find_one(self, query=None, **kwargs):
             body=elasticsearch_query,
         )['hits']['hits']
 
-        for match in matches:
-            self._to_native_types(match)
-
         if len(matches) == 1:
-            return matches[0]
+            return self._from_elastic_types(matches[0])
 
         if len(matches) == 0:
             raise NoResultsFound()
@@ -127,14 +124,22 @@ def find_one(self, query=None, **kwargs):
         )
 
     def get(self, primary_name, key):
-        match = self.client.get(index=self.es_index, doc_type=self.collection, id=key)
-        self._to_native_types(match)
-        return match
+        match = {}
+        try:
+            match = self.client.get(index=self.es_index, doc_type=self.collection, id=key)
+        except NotFoundError:
+            return None
+
+        return self._from_elastic_types(match)
 
     def insert(self, primary_name, key, value):
-        self.client.create(index=self.es_index, doc_type=self.collection, id=key, body=value)
+        self.client.create(
+            index=self.es_index, doc_type=self.collection, id=key,
+            body=self._to_elastic_types(value),
+        )
 
     def update(self, query, data):
+        data = self._to_elastic_types(data)
         for doc in self.find(query):
             self.client.update(
                 index=self.es_index,
@@ -217,5 +222,17 @@ def _build_query(self, query=None, elasticsearch_query=None):
     def _stringop_to_regex(self, operator, argument):
         return STRINGOP_MAP[operator] % argument
 
-    def _to_native_types(self, match):
+    def _to_elastic_types(self, match):
+        for foo in match:
+            if type(match[foo]) is tuple:
+                match[foo] = (str(match[foo][0]), match[foo][1])
+
+        return match
+
+    def _from_elastic_types(self, match):
+        match = match['_source']
+        for foo in match:
+            if type(match[foo]) is tuple:
+                match[foo] = (int(match[foo][0]), match[foo][1])
+
         return match

From a2a88634e3557d02dab9809dc9369dd57d8e192c Mon Sep 17 00:00:00 2001
From: Fitz Elliott <felliott@fiskur.org>
Date: Mon, 31 Mar 2014 12:10:15 -0400
Subject: [PATCH 15/23] fix linting complaints in test_simple_queries

---
 tests/queries/test_simple_queries.py | 40 ++++++++++------------------
 1 file changed, 14 insertions(+), 26 deletions(-)

diff --git a/tests/queries/test_simple_queries.py b/tests/queries/test_simple_queries.py
index 09453e0..821bf02 100644
--- a/tests/queries/test_simple_queries.py
+++ b/tests/queries/test_simple_queries.py
@@ -37,7 +37,8 @@ def test_load_by_pk(self):
         )
 
     def test_find_all(self):
-        """ If no query object is passed, ``.find()`` should return all objects.
+        """ If no query object is passed, ``.find()`` should return all
+        objects.
         """
         self.assertEqual(
             len(self.Foo.find()),
@@ -45,8 +46,8 @@ def test_find_all(self):
         )
 
     def test_find_one(self):
-        """ Given a query with exactly one result record, ``.find_one()`` should
-        return that object.
+        """ Given a query with exactly one result record, ``.find_one()``
+        should return that object.
         """
         self.assertEqual(
             self.Foo.find_one(Q('_id', 'eq', 0))._id,
@@ -54,8 +55,8 @@ def test_find_one(self):
         )
 
     def test_find_one_return_zero(self):
-        """ Given a query with zero result records, ``.find_one()`` should raise
-         an appropriate error.
+        """ Given a query with zero result records, ``.find_one()`` should
+        raise an appropriate error.
         """
         with self.assertRaises(exceptions.NoResultsFound):
             self.Foo.find_one(Q('_id', 'eq', -1))
@@ -68,9 +69,7 @@ def test_find_one_return_many(self):
             result = self.Foo.find_one()
             logger.debug(result)
 
-
     # individual filter tests (limit, offset, sort)
-
     def test_limit(self):
         """ For a query that returns > n results, `.limit(n)` should return the
          first n.
@@ -86,7 +85,6 @@ def test_limit(self):
         )
         # TODO: test limit = 0
 
-
     def test_offset(self):
         """For a query that returns n results, ``.offset(m)`` should return
         n - m results, skipping the first m that would otherwise have been
@@ -98,7 +96,6 @@ def test_offset(self):
         )
         # TODO: test offset = 0, offset > self.COUNT
 
-
     def test_sort(self):
         results = self.Foo.find().sort('-_id')
         self.assertListEqual(
@@ -106,22 +103,19 @@ def test_sort(self):
             range(self.COUNT)[::-1],
         )
 
-
     # paired filter tests:
     #   limit  + {limit,offset,sort}
     #   offset + {offset,sort}
     #   sort   + sort
     # each test sub tests the filters in both orders. i.e. limit + offset
     # tests .limit().offset() AND .offset().limit()
-
     def test_limit_limit(self):
-        self.assertEqual( len(self.Foo.find().limit(5).limit(10)), 10 )
-        self.assertEqual( len(self.Foo.find().limit(10).limit(5)), 5  )
-
+        self.assertEqual(len(self.Foo.find().limit(5).limit(10)), 10)
+        self.assertEqual(len(self.Foo.find().limit(10).limit(5)), 5)
 
     def test_limit_offset(self):
-        self.assertEqual( len(self.Foo.find().limit(2).offset(2)), 2 )
-        self.assertEqual( len(self.Foo.find().offset(2).limit(2)), 2 )
+        self.assertEqual(len(self.Foo.find().limit(2).offset(2)), 2)
+        self.assertEqual(len(self.Foo.find().offset(2).limit(2)), 2)
 
         tmp = 5
         limit = tmp + 5
@@ -129,7 +123,6 @@ def test_limit_offset(self):
         self.assertEqual(len(self.Foo.find().limit(limit).offset(offset)), tmp)
         self.assertEqual(len(self.Foo.find().offset(offset).limit(limit)), tmp)
 
-
     def test_limit_sort(self):
         limit, sort, = [10, '-_id']
         expect = range(self.COUNT-limit, self.COUNT)[::-1]
@@ -140,7 +133,6 @@ def test_limit_sort(self):
         results = self.Foo.find().sort(sort).limit(limit)
         self.assertListEqual([x._id for x in results], expect)
 
-
     def test_offset_offset(self):
         self.assertEqual(
             len(self.Foo.find().offset(10).offset(17)),
@@ -151,7 +143,6 @@ def test_offset_offset(self):
             self.COUNT-10
         )
 
-
     def test_offset_sort(self):
         offset, sort = [27, '-_id']
         expect = range(self.COUNT-offset)[::-1]
@@ -162,7 +153,6 @@ def test_offset_sort(self):
         results = self.Foo.find().sort(sort).offset(offset)
         self.assertListEqual([x._id for x in results], expect)
 
-
     def test_sort_sort(self):
         results = self.Foo.find().sort('-_id').sort('_id')
         self.assertListEqual(
@@ -175,15 +165,13 @@ def test_sort_sort(self):
             range(self.COUNT)[::-1],
         )
 
-
     # all three filters together
-
     def test_limit_offset_sort(self):
         test_sets = [
             # limit offset sort    expect
-            [ 10,   7,     '-_id', range(self.COUNT-7-10, self.COUNT-7)[::-1], ],
-            [ 20,   17,    '_id',  range(17, self.COUNT),                      ],
-            [ 10,   5,     '_id',  range(5, 5+10),                             ],
+            [10,   7,     '-_id', range(self.COUNT-7-10, self.COUNT-7)[::-1]],
+            [20,   17,    '_id',  range(17, self.COUNT)],
+            [10,   5,     '_id',  range(5, 5+10)],
         ]
         for test in test_sets:
             limit, offset, sort, expect = test
@@ -197,4 +185,4 @@ def test_limit_offset_sort(self):
             ]
 
             for result in all_combinations:
-                self.assertListEqual( [x._id for x in result], expect )
+                self.assertListEqual([x._id for x in result], expect)

From bf355de504fda6d17604360e75bff5629211560e Mon Sep 17 00:00:00 2001
From: Fitz Elliott <felliott@fiskur.org>
Date: Mon, 31 Mar 2014 12:13:02 -0400
Subject: [PATCH 16/23] make Elasticsearch evaluate lazily

---
 modularodm/storage/elasticsearchstorage.py | 43 +++++++++++++++-------
 1 file changed, 30 insertions(+), 13 deletions(-)

diff --git a/modularodm/storage/elasticsearchstorage.py b/modularodm/storage/elasticsearchstorage.py
index e576730..9eac6c7 100644
--- a/modularodm/storage/elasticsearchstorage.py
+++ b/modularodm/storage/elasticsearchstorage.py
@@ -19,21 +19,46 @@ class ElasticsearchQuerySet(BaseQuerySet):
     def __init__(self, schema, data):
         super(ElasticsearchQuerySet, self).__init__(schema)
         self.data = list(data)
+        self._sort = None
+        self._offset = None
+        self._limit = None
+
+    def _eval(self):
+        if (self._sort is not None):
+            for key in self._sort[::-1]:
+                if key.startswith('-'):
+                    reverse = True
+                    key = key.lstrip('-')
+                else:
+                    reverse = False
+
+            self.data = sorted(self.data, key=lambda record: record[key], reverse=reverse)
+
+        if (self._offset is not None):
+            self.data = self.data[self._offset:]
+
+        if (self._limit is not None):
+            self.data = self.data[:self._limit]
+
+        return self
 
     def __getitem__(self, index, raw=False):
         super(ElasticsearchQuerySet, self).__getitem__(index)
+        self._eval()
         key = self.data[index][self.primary]
         if raw:
             return key
         return self.schema.load(key)
 
     def __iter__(self, raw=False):
+        self._eval()
         keys = [obj[self.primary] for obj in self.data]
         if raw:
             return keys
         return (self.schema.load(key) for key in keys)
 
     def __len__(self):
+        self._eval()
         return len(self.data)
 
     count = __len__
@@ -45,27 +70,19 @@ def get_keys(self):
         return list(self.__iter__(raw=True))
 
     def sort(self, *keys):
-        sort_key = []
-        for key in keys[::-1]:
-
-            if key.startswith('-'):
-                reverse = True
-                key = key.lstrip('-')
-            else:
-                reverse = False
-
-            self.data = sorted(self.data, key=lambda record: record[key], reverse=reverse)
-
+        """ Iteratively sort data by keys in reverse order. """
+        self._sort = keys
         return self
 
     def offset(self, n):
-        self.data = self.data[n:]
+        self._offset = n
         return self
 
     def limit(self, n):
-        self.data = self.data[:n]
+        self._limit = n
         return self
 
+
 class ElasticsearchStorage(Storage):
 
     QuerySet = ElasticsearchQuerySet

From 61dd33ef23cdac88c9f7fb48da60568b8264bda9 Mon Sep 17 00:00:00 2001
From: Fitz Elliott <felliott@fiskur.org>
Date: Mon, 31 Mar 2014 12:13:51 -0400
Subject: [PATCH 17/23] fix lint complaints in elasticsearch storage

---
 modularodm/storage/elasticsearchstorage.py | 24 ++++++++++++++--------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/modularodm/storage/elasticsearchstorage.py b/modularodm/storage/elasticsearchstorage.py
index 9eac6c7..c137e4f 100644
--- a/modularodm/storage/elasticsearchstorage.py
+++ b/modularodm/storage/elasticsearchstorage.py
@@ -12,7 +12,12 @@
 NEGATION_OPERATORS = ('ne', 'nin')
 
 STRING_OPERATORS = ('contains', 'icontains', 'endswith')
-STRINGOP_MAP = {'contains': '.*%s.*', 'icontains': '.*%s.*', 'endswith': '.*%s',}
+STRINGOP_MAP = {
+    'contains':  '.*%s.*',
+    'icontains': '.*%s.*',
+    'endswith':  '.*%s',
+}
+
 
 class ElasticsearchQuerySet(BaseQuerySet):
 
@@ -166,8 +171,8 @@ def update(self, query, data):
 
     def remove(self, query=None):
         elasticsearch_query = self._translate_query(query)
-        delete_query = {"filtered" : {
-            "query" : {"match_all" : {}},
+        delete_query = {"filtered": {
+            "query": {"match_all": {}},
             "filter": elasticsearch_query['filter'],
         }}
         self.client.delete_by_query(
@@ -184,7 +189,7 @@ def __repr__(self):
 
     def _translate_query(self, query=None, elasticsearch_query=None):
         elasticsearch_query = self._build_query(query, elasticsearch_query)
-        return {'filter' : elasticsearch_query}
+        return {'filter': elasticsearch_query}
 
     def _build_query(self, query=None, elasticsearch_query=None):
         """Turn a query object into a valid elasticsearch filter dict"""
@@ -198,7 +203,9 @@ def _build_query(self, query=None, elasticsearch_query=None):
                 elasticsearch_query['term'] = {attribute: argument}
 
             elif operator in RANGE_OPERATORS:
-                elasticsearch_query['range'] = {attribute: {operator: argument}}
+                elasticsearch_query['range'] = {
+                    attribute: {operator: argument}
+                }
 
             elif operator in SET_OPERATORS:
                 elasticsearch_query['terms'] = {attribute: argument}
@@ -211,19 +218,18 @@ def _build_query(self, query=None, elasticsearch_query=None):
                     attribute: self._stringop_to_regex(operator, argument)
                 }
 
-
             if operator in NEGATION_OPERATORS:
                 elasticsearch_query = {"not": elasticsearch_query}
 
         elif isinstance(query, QueryGroup):
             if query.operator == 'and':
-                return {'and' : [self._build_query(node) for node in query.nodes]}
+                return {'and': [self._build_query(node) for node in query.nodes]}
 
             elif query.operator == 'or':
-                return {'or' : [self._build_query(node) for node in query.nodes]}
+                return {'or': [self._build_query(node) for node in query.nodes]}
 
             elif query.operator == 'not':
-                return {'not' : self._build_query(query.nodes[0])}
+                return {'not': self._build_query(query.nodes[0])}
 
             else:
                 raise ValueError('QueryGroup operator must be <and>, <or>, or <not>.')

From d9894986614e9f960f88de9aced1e115e10ddaed Mon Sep 17 00:00:00 2001
From: Fitz Elliott <felliott@fiskur.org>
Date: Mon, 31 Mar 2014 12:51:39 -0400
Subject: [PATCH 18/23] add NullHandler to silence ES.trace logging

---
 tests/base.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tests/base.py b/tests/base.py
index 0ca1683..76d03d4 100644
--- a/tests/base.py
+++ b/tests/base.py
@@ -10,7 +10,14 @@
 from modularodm import StoredObject
 from modularodm.storage import MongoStorage, PickleStorage, EphemeralStorage, ElasticsearchStorage
 
+
+class NullHandler(logging.Handler):
+    def emit(self, record):
+        pass
+
 logger = logging.getLogger(__name__)
+logging.getLogger('elasticsearch.trace').addHandler(NullHandler())
+
 
 class TestObject(StoredObject):
     def __init__(self, *args, **kwargs):

From 374c8df5f881e75727814191af4f1a9f13da927a Mon Sep 17 00:00:00 2001
From: Fitz Elliott <felliott@fiskur.org>
Date: Mon, 31 Mar 2014 13:20:07 -0400
Subject: [PATCH 19/23] fix foolish unpacking of ES matches

---
 modularodm/storage/elasticsearchstorage.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/modularodm/storage/elasticsearchstorage.py b/modularodm/storage/elasticsearchstorage.py
index c137e4f..d27aae6 100644
--- a/modularodm/storage/elasticsearchstorage.py
+++ b/modularodm/storage/elasticsearchstorage.py
@@ -112,10 +112,11 @@ def find(self, query=None, **kwargs):
         ):
             matches.append(results)
 
+        results = []
         for match in matches:
-            match = self._from_elastic_types(match)
+            results.append(self._from_elastic_types(match))
 
-        return matches
+        return results
 
     def find_one(self, query=None, **kwargs):
         """ Gets a single object from the collection.
@@ -253,9 +254,9 @@ def _to_elastic_types(self, match):
         return match
 
     def _from_elastic_types(self, match):
-        match = match['_source']
-        for foo in match:
-            if type(match[foo]) is tuple:
-                match[foo] = (int(match[foo][0]), match[foo][1])
+        result = match['_source']
+        for foo in result:
+            if type(result[foo]) is tuple:
+                result[foo] = (int(result[foo][0]), result[foo][1])
 
-        return match
+        return result

From 3b170cbf5bc712850389fabb36e3fc9bbb4ff5aa Mon Sep 17 00:00:00 2001
From: Fitz Elliott <felliott@fiskur.org>
Date: Mon, 31 Mar 2014 15:47:10 -0400
Subject: [PATCH 20/23] add refresh() method to Storage base

 * Elasicsearch does real-time search, so searching immediately after
   save() may not return up-to-date-results.  Add a default-noop
   refresh() method to Storage() and implement for Elasticsearch
   backend.  Call after save().
---
 modularodm/storage/base.py                 | 5 +++++
 modularodm/storage/elasticsearchstorage.py | 5 +++++
 modularodm/storedobject.py                 | 1 +
 3 files changed, 11 insertions(+)

diff --git a/modularodm/storage/base.py b/modularodm/storage/base.py
index 9b5a24d..7fc6dd2 100644
--- a/modularodm/storage/base.py
+++ b/modularodm/storage/base.py
@@ -213,3 +213,8 @@ def find(self, query=None, **kwargs):
         """Query the database and return a query set.
         """
         raise NotImplementedError
+
+    def refresh(self):
+        """Refresh the index (needed for real-time databases)
+        """
+        pass
diff --git a/modularodm/storage/elasticsearchstorage.py b/modularodm/storage/elasticsearchstorage.py
index d27aae6..2b3d452 100644
--- a/modularodm/storage/elasticsearchstorage.py
+++ b/modularodm/storage/elasticsearchstorage.py
@@ -1,3 +1,4 @@
+import elasticsearch
 from elasticsearch import helpers, NotFoundError
 
 from .base import Storage
@@ -260,3 +261,7 @@ def _from_elastic_types(self, match):
                 result[foo] = (int(result[foo][0]), result[foo][1])
 
         return result
+
+    def refresh(self):
+        indices = elasticsearch.client.IndicesClient(self.client)
+        indices.refresh(index=self.es_index)
diff --git a/modularodm/storedobject.py b/modularodm/storedobject.py
index f0ad44a..6b85368 100644
--- a/modularodm/storedobject.py
+++ b/modularodm/storedobject.py
@@ -820,6 +820,7 @@ def save(self, force=False):
         else:
             self.insert(self._primary_key, storage_data)
 
+        self._storage[0].refresh()
         # if primary key has changed, follow back references and update
         # AND
         # run after_save or after_save_on_difference

From 9dda082f660a49df3d8f459dba04665258d3672b Mon Sep 17 00:00:00 2001
From: Fitz Elliott <felliott@fiskur.org>
Date: Mon, 7 Jul 2014 07:36:55 -0400
Subject: [PATCH 21/23] require ES v1.0 or greater

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 12f36d6..5f091cf 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,4 +2,4 @@ flask
 blinker
 pymongo
 python-dateutil
-elasticsearch
+elasticsearch>=1.0

From 31c8ba096955be3a686e69886b36d47693a43dc1 Mon Sep 17 00:00:00 2001
From: Fitz Elliott <felliott@fiskur.org>
Date: Mon, 7 Jul 2014 07:37:07 -0400
Subject: [PATCH 22/23] add option to tasks.py to run ES as daemon

---
 tasks.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tasks.py b/tasks.py
index 4c5379b..66b84ef 100644
--- a/tasks.py
+++ b/tasks.py
@@ -15,10 +15,12 @@ def mongo(daemon=False, port=20771):
     run(cmd)
 
 @task
-def elastic(port=20772, clustername='modular-odm-test'):
+def elastic(daemon=False, port=20772, clustername='modular-odm-test'):
     '''Run the elasticsearch process.
     '''
     cmd = "elasticsearch -Des.http.port={0} -Des.cluster.name={1}".format(port, clustername)
+    if daemon:
+        cmd += " -d"
     run(cmd)
 
 @task

From 2b50b38d7c4246428d489a11558a714454de7a71 Mon Sep 17 00:00:00 2001
From: Fitz Elliott <felliott@fiskur.org>
Date: Mon, 7 Jul 2014 07:21:29 -0400
Subject: [PATCH 23/23] update ES delete syntax for v1.0

---
 modularodm/storage/elasticsearchstorage.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modularodm/storage/elasticsearchstorage.py b/modularodm/storage/elasticsearchstorage.py
index 2b3d452..05ffdce 100644
--- a/modularodm/storage/elasticsearchstorage.py
+++ b/modularodm/storage/elasticsearchstorage.py
@@ -173,10 +173,10 @@ def update(self, query, data):
 
     def remove(self, query=None):
         elasticsearch_query = self._translate_query(query)
-        delete_query = {"filtered": {
+        delete_query = {"query": {"filtered": {
             "query": {"match_all": {}},
             "filter": elasticsearch_query['filter'],
-        }}
+        }}}
         self.client.delete_by_query(
             index=self.es_index,
             doc_type=self.collection,