diff --git a/Makefile b/Makefile
index f8c457e4..300aca34 100644
--- a/Makefile
+++ b/Makefile
@@ -4,6 +4,9 @@ envtest: clean
test:
tox
+tests_nose:
+ nosetests -dsv --with-yanc --with-coverage --cover-package rows tests/*.py
+
clean:
find -regex '.*\.pyc' -exec rm {} \;
find -regex '.*~' -exec rm {} \;
@@ -31,7 +34,11 @@ man:
head -1 rows.1.txt > rows.1
txt2man rows.1.txt | egrep -v '^\.TH' >> rows.1
+dev-setup:
+ pip install --editable .[all]
+ pip install -r requirements-development.txt
+
release:
python setup.py bdist bdist_wheel bdist_egg upload
-.PHONY: test clean lint lint-tests install uninstall man release
+.PHONY: test clean lint lint-tests install uninstall man release
\ No newline at end of file
diff --git a/rows/__init__.py b/rows/__init__.py
index 2ad57647..1f520fc4 100644
--- a/rows/__init__.py
+++ b/rows/__init__.py
@@ -27,6 +27,9 @@
# Don't have dependencies or dependencies installed on `install_requires`
+import_from_yaml = plugins.yaml.import_from_yaml
+export_to_yaml = plugins.yaml.export_to_yaml
+
import_from_json = plugins.json.import_from_json
export_to_json = plugins.json.export_to_json
diff --git a/rows/plugins/__init__.py b/rows/plugins/__init__.py
index 3a2a307c..40df5f01 100644
--- a/rows/plugins/__init__.py
+++ b/rows/plugins/__init__.py
@@ -16,6 +16,7 @@
# along with this program. If not, see .
from . import plugin_json as json
+from . import plugin_yaml as yaml
from . import dicts as dicts
from . import plugin_csv as csv
from . import txt as txt
diff --git a/rows/plugins/plugin_json.py b/rows/plugins/plugin_json.py
index 795b990e..5dbf14d7 100644
--- a/rows/plugins/plugin_json.py
+++ b/rows/plugins/plugin_json.py
@@ -17,10 +17,7 @@
from __future__ import unicode_literals
-import datetime
-import decimal
import json
-
import six
from rows import fields
diff --git a/rows/plugins/plugin_yaml.py b/rows/plugins/plugin_yaml.py
new file mode 100644
index 00000000..dd5c394a
--- /dev/null
+++ b/rows/plugins/plugin_yaml.py
@@ -0,0 +1,85 @@
+# coding: utf-8
+
+# Copyright 2014-2016 Álvaro Justen
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+from __future__ import unicode_literals
+
+import six
+import yaml
+
+from rows import fields
+from rows.plugins.utils import (create_table, export_data,
+ get_filename_and_fobj, prepare_to_export)
+
+
+def import_from_yaml(filename_or_fobj, encoding='utf-8', *args, **kwargs):
+ '''Import a YAML file or file-like object into a `rows.Table`
+
+ If a file-like object is provided it MUST be open in text (non-binary) mode
+ on Python 3 and could be open in both binary or text mode on Python 2.
+ '''
+
+ filename, fobj = get_filename_and_fobj(filename_or_fobj)
+
+ yaml_obj = yaml.load(fobj)
+ field_names = list(yaml_obj[0].keys())
+ table_rows = [[item[key] for key in field_names] for item in yaml_obj]
+
+ meta = {
+ 'imported_from': 'yaml',
+ 'filename': filename,
+ 'encoding': encoding
+ }
+ return create_table([field_names] + table_rows, meta=meta, *args, **kwargs)
+
+
+def _convert(value, field_type, *args, **kwargs):
+ if value is None or field_type in (
+ fields.BinaryField,
+ fields.BoolField,
+ fields.FloatField,
+ fields.IntegerField,
+ fields.JSONField,
+ fields.TextField,):
+ return value
+ else:
+ return field_type.serialize(value, *args, **kwargs)
+
+
+def export_to_yaml(table, filename_or_fobj=None, encoding='utf-8', indent=None,
+ *args, **kwargs):
+ '''Export a `rows.Table` to a YAML file or file-like object
+ '''
+
+ all_fields = table.fields
+ prepared_table = prepare_to_export(table, *args, **kwargs)
+ field_names = next(prepared_table)
+ data = [{field_name: _convert(value,
+ all_fields[field_name],
+ *args,
+ **kwargs)
+ for field_name, value in zip(field_names, row)}
+ for row in prepared_table]
+
+ result = yaml.dump(data, indent=indent)
+ if type(result) is six.text_type:
+ result = result.encode(encoding)
+
+ if indent is not None:
+ # clean up empty spaces at the end of lines
+ result = b'\n'.join(line.rstrip() for line in result.splitlines())
+
+ return export_data(filename_or_fobj, result, mode='wb')
diff --git a/setup.py b/setup.py
index 0e5afcaf..f026d76f 100644
--- a/setup.py
+++ b/setup.py
@@ -22,6 +22,7 @@
EXTRA_REQUIREMENTS = {
'csv': ['unicodecsv'],
+ 'yaml': ['pyyaml'],
'cli': ['click', 'requests'],
'html': ['lxml'], # apt: libxslt-dev libxml2-dev
'ods': ['lxml'],
diff --git a/tests/data/all-field-types.yaml b/tests/data/all-field-types.yaml
new file mode 100644
index 00000000..0dadcce8
--- /dev/null
+++ b/tests/data/all-field-types.yaml
@@ -0,0 +1,57 @@
+---
+- float_column: 3.141592
+ decimal_column: 3.141592
+ bool_column: 'True'
+ integer_column: 1
+ date_column: '2015-01-01'
+ datetime_column: '2015-08-18T15:10:00'
+ percent_column: 1%
+ unicode_column: Álvaro
+- float_column: 1.234
+ decimal_column: 1.234
+ bool_column: 'False'
+ integer_column: 2
+ date_column: '1999-02-03'
+ datetime_column: '1999-02-03T00:01:02'
+ percent_column: 11.69%
+ unicode_column: àáãâä¹²³
+- float_column: 4.56
+ decimal_column: 4.56
+ bool_column: true
+ integer_column: 3
+ date_column: '2050-01-02'
+ datetime_column: '2050-01-02T23:45:31'
+ percent_column: 12%
+ unicode_column: éèẽêë
+- float_column: 7.89
+ decimal_column: 7.89
+ bool_column: false
+ integer_column: 4
+ date_column: '2015-08-18'
+ datetime_column: '2015-08-18T22:21:33'
+ percent_column: 13.64%
+ unicode_column: "~~~~"
+- float_column: 9.87
+ decimal_column: 9.87
+ bool_column: 'yes'
+ integer_column: 5
+ date_column: '2015-03-04'
+ datetime_column: '2015-03-04T16:00:01'
+ percent_column: 13.14%
+ unicode_column: álvaro
+- float_column: 1.2345
+ decimal_column: 1.2345
+ bool_column: 'no'
+ integer_column: 6
+ date_column: '2015-05-06'
+ datetime_column: '2015-05-06T12:01:02'
+ percent_column: 2%
+ unicode_column: test
+- float_column: ''
+ decimal_column: "-"
+ bool_column: 'null'
+ integer_column: nil
+ date_column: none
+ datetime_column: n/a
+ percent_column: 'null'
+ unicode_column: ''
diff --git a/tests/tests_plugin_yaml.py b/tests/tests_plugin_yaml.py
new file mode 100644
index 00000000..e593ec89
--- /dev/null
+++ b/tests/tests_plugin_yaml.py
@@ -0,0 +1,181 @@
+# coding: utf-8
+
+# Copyright 2014-2016 Álvaro Justen
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+from __future__ import unicode_literals
+
+import unittest
+import tempfile
+import yaml
+
+from collections import Counter
+from collections import OrderedDict
+from collections import defaultdict
+
+import six
+import mock
+
+import rows
+import tests.utils as utils
+
+
+class PluginYamlTestCase(utils.RowsTestMixIn, unittest.TestCase):
+
+ plugin_name = 'yaml'
+ file_extension = 'yaml'
+ filename = 'tests/data/all-field-types.yaml'
+ encoding = 'utf-8'
+ assert_meta_encoding = True
+
+ def test_imports(self):
+ self.assertIs(rows.import_from_yaml,
+ rows.plugins.plugin_yaml.import_from_yaml)
+ self.assertIs(rows.export_to_yaml,
+ rows.plugins.plugin_yaml.export_to_yaml)
+
+ @mock.patch('rows.plugins.plugin_yaml.create_table')
+ def test_import_from_yaml_uses_create_table(self, mocked_create_table):
+ mocked_create_table.return_value = 42
+ kwargs = {'some_key': 123, 'other': 456, }
+ result = rows.import_from_yaml(self.filename, encoding=self.encoding,
+ **kwargs)
+ self.assertTrue(mocked_create_table.called)
+ self.assertEqual(mocked_create_table.call_count, 1)
+ self.assertEqual(result, 42)
+
+ call = mocked_create_table.call_args
+ kwargs['meta'] = {'imported_from': 'yaml',
+ 'filename': self.filename,
+ 'encoding': self.encoding,}
+ self.assertEqual(call[1], kwargs)
+
+ @mock.patch('rows.plugins.plugin_yaml.create_table')
+ def test_import_from_yaml_retrieve_desired_data(self, mocked_create_table):
+ mocked_create_table.return_value = 42
+
+ # import using filename
+ table_1 = rows.import_from_yaml(self.filename)
+ call_args = mocked_create_table.call_args_list[0]
+ self.assert_create_table_data(call_args, field_ordering=False)
+
+ # import using fobj
+ with open(self.filename) as fobj:
+ table_2 = rows.import_from_yaml(fobj)
+ call_args = mocked_create_table.call_args_list[1]
+ self.assert_create_table_data(call_args, field_ordering=False)
+
+ @mock.patch('rows.plugins.plugin_yaml.create_table')
+ def test_import_from_yaml_uses_create_table(self, mocked_create_table):
+ mocked_create_table.return_value = 42
+ kwargs = {'some_key': 123, 'other': 456, }
+ encoding = 'iso-8859-15'
+ result = rows.import_from_yaml(self.filename, encoding=encoding,
+ **kwargs)
+ self.assertTrue(mocked_create_table.called)
+ self.assertEqual(mocked_create_table.call_count, 1)
+ self.assertEqual(result, 42)
+
+ call = mocked_create_table.call_args
+ kwargs['meta'] = {'imported_from': 'yaml',
+ 'filename': self.filename,
+ 'encoding': encoding,}
+ self.assertEqual(call[1], kwargs)
+
+ @mock.patch('rows.plugins.plugin_yaml.prepare_to_export')
+ def test_export_to_yaml_uses_prepare_to_export(self,
+ mocked_prepare_to_export):
+ temp = tempfile.NamedTemporaryFile(delete=False, mode='wb')
+ self.files_to_delete.append(temp.name)
+ kwargs = {'test': 123, 'parameter': 3.14, }
+ mocked_prepare_to_export.return_value = \
+ iter([utils.table.fields.keys()])
+
+ rows.export_to_yaml(utils.table, temp.name, **kwargs)
+ self.assertTrue(mocked_prepare_to_export.called)
+ self.assertEqual(mocked_prepare_to_export.call_count, 1)
+
+ call = mocked_prepare_to_export.call_args
+ self.assertEqual(call[0], (utils.table, ))
+ self.assertEqual(call[1], kwargs)
+
+ @mock.patch('rows.plugins.plugin_yaml.export_data')
+ def test_export_to_yaml_uses_export_data(self, mocked_export_data):
+ temp = tempfile.NamedTemporaryFile(delete=False, mode='wb')
+ self.files_to_delete.append(temp.name)
+ kwargs = {'test': 123, 'parameter': 3.14, }
+ mocked_export_data.return_value = 42
+
+ result = rows.export_to_yaml(utils.table, temp.name, **kwargs)
+ self.assertTrue(mocked_export_data.called)
+ self.assertEqual(mocked_export_data.call_count, 1)
+ self.assertEqual(result, 42)
+
+ call = mocked_export_data.call_args
+ self.assertEqual(call[0][0], temp.name)
+ self.assertEqual(call[1], {'mode': 'wb'})
+
+ def test_export_to_yaml_filename(self):
+ # TODO: may test file contents
+ temp = tempfile.NamedTemporaryFile(delete=False, mode='wb')
+ self.files_to_delete.append(temp.name)
+ rows.export_to_yaml(utils.table, temp.name)
+ table = rows.import_from_yaml(temp.name)
+ self.assert_table_equal(table, utils.table)
+
+ def test_export_to_yaml_fobj(self):
+ # TODO: may test with codecs.open passing an encoding
+ # TODO: may test file contents
+ temp = tempfile.NamedTemporaryFile(delete=False, mode='wb')
+ self.files_to_delete.append(temp.name)
+ rows.export_to_yaml(utils.table, temp.file)
+
+ table = rows.import_from_yaml(temp.name)
+ self.assert_table_equal(table, utils.table)
+
+ def test_export_to_yaml_filename_save_data_in_correct_format(self):
+ temp = tempfile.NamedTemporaryFile(delete=False, mode='wb')
+ self.files_to_delete.append(temp.name)
+
+ rows.export_to_yaml(utils.table, temp.name)
+
+ with open(temp.name) as fobj:
+ imported_yaml = yaml.load(fobj)
+
+ COLUMN_TYPE = {
+ 'float_column': float,
+ 'decimal_column': float,
+ 'bool_column': bool,
+ 'integer_column': int,
+ 'date_column': six.text_type,
+ 'datetime_column': six.text_type,
+ 'percent_column': six.text_type,
+ 'unicode_column': six.text_type,
+ }
+ field_types = defaultdict(list)
+ for row in imported_yaml:
+ for field_name, value in row.items():
+ field_types[field_name].append(type(value))
+ # We test if the JSON was created serializing all the fields correctly
+ # (some as native JSON values, like int and float) and others needed to
+ # be serialized, like date, datetime etc.
+ for field_name, value_types in field_types.items():
+ if field_name != 'unicode_column':
+ self.assertEqual(Counter(value_types),
+ Counter({type(None): 1,
+ COLUMN_TYPE[field_name]: 6}))
+ else:
+ self.assertEqual(Counter(value_types),
+ Counter({COLUMN_TYPE[field_name]: 7}))