From 0d378f8258c16d4ac72a4cae7643ec96aaa86a10 Mon Sep 17 00:00:00 2001
From: Kartik Prabhu <me@kartikprabhu.com>
Date: Tue, 20 Feb 2018 21:03:01 -0500
Subject: [PATCH 01/11] Squashed commit of the following:

commit d0ea9ea9ef2d3cb8839bc66a1b1aef20f2121529
Author: Kartik Prabhu <me@kartikprabhu.com>
Date:   Tue Feb 20 20:56:09 2018 -0500

    bump version to 1.0.6

commit 9be5edb8cca285a89fbd3abb210403b212260520
Author: Kartik Prabhu <me@kartikprabhu.com>
Date:   Tue Feb 20 20:52:24 2018 -0500

    fix #59 #57 #43 . use html5lib by default unless user-specified a html parser. If neither exist then default to BS4 to decide. use final redirect URL while requesting from server.

commit 3995eacabf00001c6d9b18ba5c43d59513bc792a
Author: Kartik Prabhu <me@kartikprabhu.com>
Date:   Sat Feb 17 22:10:19 2018 -0500

    fix #55; parse title from link element for p-*. parse poster from video element for u-*
---
 mf2py/parse_property.py | 48 +++++++++++++--------------------
 mf2py/parser.py         | 60 ++++++++++++++++++++++++++++-------------
 mf2py/version.py        |  2 +-
 3 files changed, 61 insertions(+), 49 deletions(-)

diff --git a/mf2py/parse_property.py b/mf2py/parse_property.py
index 8ee4a37..858f0df 100644
--- a/mf2py/parse_property.py
+++ b/mf2py/parse_property.py
@@ -39,40 +39,31 @@ def get_vcp_children(el):
 
 def text(el):
     """Process p-* properties"""
+
     # handle value-class-pattern
     value_els = get_vcp_children(el)
     if value_els:
         return ''.join(get_vcp_value(el) for el in value_els)
 
-    prop_value = get_attr(el, "title", check_name="abbr")
-    if prop_value is not None:
-        return prop_value
-
-    prop_value = get_attr(el, "value", check_name=("data", "input"))
-    if prop_value is not None:
-        return prop_value
-
-    prop_value = get_attr(el, "alt", check_name=("img", "area"))
-    if prop_value is not None:
-        return prop_value
+    prop_value = get_attr(el, "title", check_name=("abbr", "link"))\
+        or get_attr(el, "value", check_name=("data", "input"))\
+        or get_attr(el, "alt", check_name=("img", "area"))\
+        or el.get_text()
 
-    # see if get_text() replaces img with alts
-    # strip here?
-    return el.get_text()
+    # drop <script> and <style>
+    # replace nested <img> with alt or src
+    # strip here
+    return prop_value
 
 
 def url(el, base_url=''):
     """Process u-* properties"""
-    prop_value = get_attr(el, "href", check_name=("a", "area", "link"))
-    if prop_value is not None:
-        return urljoin(base_url, prop_value)  # make urls absolute
 
-    prop_value = get_attr(el, "src", check_name=("img", "audio", "video",
-                                                 "source"))
-    if prop_value is not None:
-        return urljoin(base_url, prop_value)
+    prop_value = get_attr(el, "href", check_name=("a", "area", "link"))\
+        or get_attr(el, "src", check_name=("img", "audio", "video", "source"))\
+        or get_attr(el, "poster", check_name="video")\
+        or get_attr(el, "data", check_name="object")
 
-    prop_value = get_attr(el, "data", check_name="object")
     if prop_value is not None:
         return urljoin(base_url, prop_value)
 
@@ -81,16 +72,13 @@ def url(el, base_url=''):
         return urljoin(base_url, ''.join(get_vcp_value(el)
                        for el in value_els))
 
-    prop_value = get_attr(el, "title", check_name="abbr")
-    if prop_value is not None:
-        return prop_value
-
-    prop_value = get_attr(el, "value", check_name=("data", "input"))
-    if prop_value is not None:
-        return prop_value
+    prop_value = get_attr(el, "title", check_name="abbr")\
+        or get_attr(el, "value", check_name=("data", "input"))\
+        or el.get_text()
 
+    # drop <script> and <style>
     # strip here?
-    return el.get_text()
+    return prop_value
 
 
 def datetime(el, default_date=None):
diff --git a/mf2py/parser.py b/mf2py/parser.py
index 743cd16..d9b6804 100644
--- a/mf2py/parser.py
+++ b/mf2py/parser.py
@@ -1,6 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals, print_function
-from bs4 import BeautifulSoup
+from bs4 import BeautifulSoup, FeatureNotFound
 from bs4.element import Tag
 from mf2py import backcompat, mf2_classes, implied_properties, parse_property
 from mf2py import temp_fixes
@@ -63,7 +63,7 @@ class Parser(object):
 
     dict_class = dict
 
-    def __init__(self, doc=None, url=None, html_parser=None):
+    def __init__(self, doc=None, url=None, html_parser='html5lib'):
         self.__url__ = None
         self.__doc__ = None
         self.__parsed__ = self.dict_class([
@@ -72,13 +72,6 @@ def __init__(self, doc=None, url=None, html_parser=None):
             ('rel-urls', self.dict_class()),
         ])
 
-        if doc is not None:
-            self.__doc__ = doc
-            if isinstance(doc, BeautifulSoup) or isinstance(doc, Tag):
-                self.__doc__ = doc
-            else:
-                self.__doc__ = BeautifulSoup(doc, features=html_parser)
-
         if url is not None:
             self.__url__ = url
 
@@ -87,13 +80,28 @@ def __init__(self, doc=None, url=None, html_parser=None):
                     'User-Agent': self.useragent,
                 })
 
-                # check for charater encodings and use 'correct' data
+                # update to final URL after redirects
+                self.__url__ = data.url
+
+                # HACK: check for character encodings and use 'correct' data
                 if 'charset' in data.headers.get('content-type', ''):
-                    self.__doc__ = BeautifulSoup(data.text,
-                                                 features=html_parser)
+                    doc = data.text
                 else:
-                    self.__doc__ = BeautifulSoup(data.content,
-                                                 features=html_parser)
+                    doc = data.content
+
+        if doc is not None:
+            self.__doc__ = doc
+            if isinstance(doc, BeautifulSoup) or isinstance(doc, Tag):
+                self.__doc__ = doc
+            else:
+                try:
+                    # try the user-given html parser or default html5lib
+                    self.__doc__ = BeautifulSoup(doc, features=html_parser)
+                except FeatureNotFound:
+                    # maybe raise a warning?
+                    # else switch to default use
+                    self.__doc__ = BeautifulSoup(doc)
+
 
         # check for <base> tag
         if self.__doc__:
@@ -134,15 +142,18 @@ def handle_microformat(root_class_names, el, value_property=None,
             properties = self.dict_class()
             children = []
             self._default_date = None
+            # flag for processing implied name
+            do_implied_name = True
 
             # parse for properties and children
             for child in get_children(el):
-                child_props, child_children = parse_props(child)
+                child_props, child_children, child_stops_implied_name = parse_props(child)
                 for key, new_value in child_props.items():
                     prop_value = properties.get(key, [])
                     prop_value.extend(new_value)
                     properties[key] = prop_value
                 children.extend(child_children)
+                do_implied_name = do_implied_name and not child_stops_implied_name
 
             # complex h-* objects can take their "value" from the
             # first explicit property ("name" for p-* or "url" for u-*)
@@ -150,7 +161,9 @@ def handle_microformat(root_class_names, el, value_property=None,
                 simple_value = properties[value_property][0]
 
             # if some properties not already found find in implied ways
-            if "name" not in properties:
+
+            # stop implied name if any p-*, e-*, h-* is already found
+            if "name" not in properties and do_implied_name:
                 properties["name"] = [text_type(prop)
                                       for prop
                                       in implied_properties.name(el)]
@@ -201,6 +214,8 @@ def parse_props(el):
             """
             props = self.dict_class()
             children = []
+            # Does this element stop implied name?
+            stops_implied_name = False
 
             classes = el.get("class", [])
             # Is this element a microformat root?
@@ -212,13 +227,16 @@ def parse_props(el):
             p_value = None
             for prop_name in mf2_classes.text(classes):
                 is_property_el = True
+                stops_implied_name = True
                 prop_value = props.setdefault(prop_name, [])
 
                 # if value has not been parsed then parse it
                 if p_value is None:
                     p_value = text_type(parse_property.text(el).strip())
 
+
                 if root_class_names:
+                    stops_implied_name = True
                     prop_value.append(handle_microformat(
                         root_class_names, el, value_property="name",
                         simple_value=p_value))
@@ -236,6 +254,7 @@ def parse_props(el):
                     u_value = parse_property.url(el, base_url=self.__url__)
 
                 if root_class_names:
+                    stops_implied_name = True
                     prop_value.append(handle_microformat(
                         root_class_names, el, value_property="url",
                         simple_value=u_value))
@@ -257,6 +276,7 @@ def parse_props(el):
                         self._default_date = new_date
 
                 if root_class_names:
+                    stops_implied_name = True
                     prop_value.append(handle_microformat(
                         root_class_names, el,
                         simple_value=text_type(dt_value)))
@@ -268,6 +288,7 @@ def parse_props(el):
             e_value = None
             for prop_name in mf2_classes.embedded(classes):
                 is_property_el = True
+                stops_implied_name = True
                 prop_value = props.setdefault(prop_name, [])
 
                 # if value has not been parsed then parse it
@@ -275,6 +296,7 @@ def parse_props(el):
                     e_value = parse_property.embedded(el)
 
                 if root_class_names:
+                    stops_implied_name = True
                     prop_value.append(handle_microformat(
                         root_class_names, el, simple_value=e_value))
                 else:
@@ -283,19 +305,21 @@ def parse_props(el):
             # if this is not a property element, but it is a h-* microformat,
             # add it to our list of children
             if not is_property_el and root_class_names:
+                stops_implied_name = True
                 children.append(handle_microformat(root_class_names, el))
 
             # parse child tags, provided this isn't a microformat root-class
             if not root_class_names:
                 for child in get_children(el):
-                    child_properties, child_microformats = parse_props(child)
+                    child_properties, child_microformats, child_stops_implied_name = parse_props(child)
                     for prop_name in child_properties:
                         v = props.get(prop_name, [])
                         v.extend(child_properties[prop_name])
                         props[prop_name] = v
                     children.extend(child_microformats)
+                    stops_implied_name = stops_implied_name or child_stops_implied_name
 
-            return props, children
+            return props, children, stops_implied_name
 
         def parse_rels(el):
             """Parse an element for rel microformats
diff --git a/mf2py/version.py b/mf2py/version.py
index f3334eb..238cedd 100644
--- a/mf2py/version.py
+++ b/mf2py/version.py
@@ -1,4 +1,4 @@
 # Define the version number. This class is exec'd by setup.py to read
 # the value without loading mf2py (loading mf2py is bad if its dependencies
 # haven't been installed yet, which is common during setup)
-__version__ = '1.0.5'
+__version__ = '1.0.6'

From e6fd377755805a91ebc31a2b639f7500fa0ef9fe Mon Sep 17 00:00:00 2001
From: Kartik Prabhu <me@kartikprabhu.com>
Date: Sat, 24 Feb 2018 14:12:35 -0500
Subject: [PATCH 02/11] remove stop-implied-name from this version

---
 mf2py/parser.py | 22 ++++------------------
 1 file changed, 4 insertions(+), 18 deletions(-)

diff --git a/mf2py/parser.py b/mf2py/parser.py
index d9b6804..59d8c84 100644
--- a/mf2py/parser.py
+++ b/mf2py/parser.py
@@ -142,18 +142,15 @@ def handle_microformat(root_class_names, el, value_property=None,
             properties = self.dict_class()
             children = []
             self._default_date = None
-            # flag for processing implied name
-            do_implied_name = True
 
             # parse for properties and children
             for child in get_children(el):
-                child_props, child_children, child_stops_implied_name = parse_props(child)
+                child_props, child_children = parse_props(child)
                 for key, new_value in child_props.items():
                     prop_value = properties.get(key, [])
                     prop_value.extend(new_value)
                     properties[key] = prop_value
                 children.extend(child_children)
-                do_implied_name = do_implied_name and not child_stops_implied_name
 
             # complex h-* objects can take their "value" from the
             # first explicit property ("name" for p-* or "url" for u-*)
@@ -162,8 +159,7 @@ def handle_microformat(root_class_names, el, value_property=None,
 
             # if some properties not already found find in implied ways
 
-            # stop implied name if any p-*, e-*, h-* is already found
-            if "name" not in properties and do_implied_name:
+            if "name" not in properties:
                 properties["name"] = [text_type(prop)
                                       for prop
                                       in implied_properties.name(el)]
@@ -214,8 +210,6 @@ def parse_props(el):
             """
             props = self.dict_class()
             children = []
-            # Does this element stop implied name?
-            stops_implied_name = False
 
             classes = el.get("class", [])
             # Is this element a microformat root?
@@ -227,7 +221,6 @@ def parse_props(el):
             p_value = None
             for prop_name in mf2_classes.text(classes):
                 is_property_el = True
-                stops_implied_name = True
                 prop_value = props.setdefault(prop_name, [])
 
                 # if value has not been parsed then parse it
@@ -236,7 +229,6 @@ def parse_props(el):
 
 
                 if root_class_names:
-                    stops_implied_name = True
                     prop_value.append(handle_microformat(
                         root_class_names, el, value_property="name",
                         simple_value=p_value))
@@ -254,7 +246,6 @@ def parse_props(el):
                     u_value = parse_property.url(el, base_url=self.__url__)
 
                 if root_class_names:
-                    stops_implied_name = True
                     prop_value.append(handle_microformat(
                         root_class_names, el, value_property="url",
                         simple_value=u_value))
@@ -276,7 +267,6 @@ def parse_props(el):
                         self._default_date = new_date
 
                 if root_class_names:
-                    stops_implied_name = True
                     prop_value.append(handle_microformat(
                         root_class_names, el,
                         simple_value=text_type(dt_value)))
@@ -288,7 +278,6 @@ def parse_props(el):
             e_value = None
             for prop_name in mf2_classes.embedded(classes):
                 is_property_el = True
-                stops_implied_name = True
                 prop_value = props.setdefault(prop_name, [])
 
                 # if value has not been parsed then parse it
@@ -296,7 +285,6 @@ def parse_props(el):
                     e_value = parse_property.embedded(el)
 
                 if root_class_names:
-                    stops_implied_name = True
                     prop_value.append(handle_microformat(
                         root_class_names, el, simple_value=e_value))
                 else:
@@ -305,21 +293,19 @@ def parse_props(el):
             # if this is not a property element, but it is a h-* microformat,
             # add it to our list of children
             if not is_property_el and root_class_names:
-                stops_implied_name = True
                 children.append(handle_microformat(root_class_names, el))
 
             # parse child tags, provided this isn't a microformat root-class
             if not root_class_names:
                 for child in get_children(el):
-                    child_properties, child_microformats, child_stops_implied_name = parse_props(child)
+                    child_properties, child_microformats = parse_props(child)
                     for prop_name in child_properties:
                         v = props.get(prop_name, [])
                         v.extend(child_properties[prop_name])
                         props[prop_name] = v
                     children.extend(child_microformats)
-                    stops_implied_name = stops_implied_name or child_stops_implied_name
 
-            return props, children, stops_implied_name
+            return props, children
 
         def parse_rels(el):
             """Parse an element for rel microformats

From 146d241c14a307a9ef4de334f6e62cf8670e46b0 Mon Sep 17 00:00:00 2001
From: Kartik Prabhu <me@kartikprabhu.com>
Date: Sat, 24 Feb 2018 15:56:44 -0500
Subject: [PATCH 03/11] fix #61. add img alt or src in the p-* and e-* value
 parsing. drop script and style from text content

---
 mf2py/dom_helpers.py        | 33 ++++++++++++++++++++++++++++++++-
 mf2py/implied_properties.py |  6 +++---
 mf2py/parse_property.py     | 19 +++++++------------
 mf2py/parser.py             |  6 +++---
 4 files changed, 45 insertions(+), 19 deletions(-)

diff --git a/mf2py/dom_helpers.py b/mf2py/dom_helpers.py
index de361cf..f94b924 100644
--- a/mf2py/dom_helpers.py
+++ b/mf2py/dom_helpers.py
@@ -1,14 +1,45 @@
 import sys
 import bs4
+import copy
 
 if sys.version < '3':
+    from urlparse import urljoin
     text_type = unicode
     binary_type = str
 else:
+    from urllib.parse import urljoin
     text_type = str
     binary_type = bytes
-    basestring = str
 
+def get_textContent(el, replace_img=False, base_url=''):
+    """ Get the text content of an element, replacing images by alt or src
+    """
+
+    # copy el to avoid making direct changes
+    el_copy = copy.deepcopy(el)
+
+    # drop all <style> and <script> elements
+    drops = el_copy.find_all(['style', 'script'])
+    for drop in drops:
+        drop.decompose()
+
+    # replace <img> with alt or src
+    if replace_img:
+        imgs = el_copy.find_all('img')
+
+        for img in imgs:
+            replacement = img.get('alt')
+            if replacement is None:
+                replacement = img.get('src')
+                if replacement is not None:
+                    replacement = ' ' + urljoin(base_url, replacement) + ' '
+
+            if replacement is None:
+                replacement = ''
+
+            img.replace_with(replacement)
+
+    return el_copy.get_text().strip()
 
 def get_attr(el, attr, check_name=None):
     """Get the attribute of an element if it exists and is not empty.
diff --git a/mf2py/implied_properties.py b/mf2py/implied_properties.py
index e633856..a59868c 100644
--- a/mf2py/implied_properties.py
+++ b/mf2py/implied_properties.py
@@ -1,6 +1,6 @@
 from __future__ import unicode_literals, print_function
 from . import mf2_classes
-from .dom_helpers import get_attr, get_children
+from .dom_helpers import get_attr, get_children, get_textContent
 import sys
 
 if sys.version < '3':
@@ -9,7 +9,7 @@
     from urllib.parse import urljoin
 
 
-def name(el):
+def name(el, base_url=''):
     """Find an implied name property
 
     Args:
@@ -60,7 +60,7 @@ def non_empty(val):
                 return [prop_value]
 
     # use text if all else fails
-    return [el.get_text().strip()]
+    return [get_textContent(el, replace_img=True, base_url=base_url)]
 
 
 def photo(el, base_url=''):
diff --git a/mf2py/parse_property.py b/mf2py/parse_property.py
index 858f0df..5eb2329 100644
--- a/mf2py/parse_property.py
+++ b/mf2py/parse_property.py
@@ -1,7 +1,7 @@
 """functions to parse the properties of elements"""
 from __future__ import unicode_literals, print_function
 
-from .dom_helpers import get_attr, get_children
+from .dom_helpers import get_attr, get_children, get_textContent
 import sys
 import re
 
@@ -37,7 +37,7 @@ def get_vcp_children(el):
             and ('value' in c['class'] or 'value-title' in c['class'])]
 
 
-def text(el):
+def text(el, base_url=''):
     """Process p-* properties"""
 
     # handle value-class-pattern
@@ -48,11 +48,8 @@ def text(el):
     prop_value = get_attr(el, "title", check_name=("abbr", "link"))\
         or get_attr(el, "value", check_name=("data", "input"))\
         or get_attr(el, "alt", check_name=("img", "area"))\
-        or el.get_text()
+        or get_textContent(el, replace_img=True, base_url=base_url)
 
-    # drop <script> and <style>
-    # replace nested <img> with alt or src
-    # strip here
     return prop_value
 
 
@@ -74,10 +71,8 @@ def url(el, base_url=''):
 
     prop_value = get_attr(el, "title", check_name="abbr")\
         or get_attr(el, "value", check_name=("data", "input"))\
-        or el.get_text()
+        or get_textContent(el)
 
-    # drop <script> and <style>
-    # strip here?
     return prop_value
 
 
@@ -172,7 +167,7 @@ def try_normalize(dtstr, match=None):
     prop_value = get_attr(el, "datetime", check_name=("time", "ins", "del"))\
         or get_attr(el, "title", check_name="abbr")\
         or get_attr(el, "value", check_name=("data", "input"))\
-        or el.get_text()  # strip here?
+        or get_textContent(el) 
 
     # if this is just a time, augment with default date
     match = re.match(TIME_RE + '$', prop_value)
@@ -186,9 +181,9 @@ def try_normalize(dtstr, match=None):
             match and match.group('date'),)
 
 
-def embedded(el):
+def embedded(el, base_url=''):
     """Process e-* properties"""
     return {
         'html': el.decode_contents(),    # secret bs4 method to get innerHTML
-        'value': el.get_text()     # strip here?
+        'value': get_textContent(el, replace_img=True, base_url=base_url)
     }
diff --git a/mf2py/parser.py b/mf2py/parser.py
index 59d8c84..dd88689 100644
--- a/mf2py/parser.py
+++ b/mf2py/parser.py
@@ -162,7 +162,7 @@ def handle_microformat(root_class_names, el, value_property=None,
             if "name" not in properties:
                 properties["name"] = [text_type(prop)
                                       for prop
-                                      in implied_properties.name(el)]
+                                      in implied_properties.name(el, base_url=self.__url__)]
             if "photo" not in properties:
                 x = implied_properties.photo(el, base_url=self.__url__)
                 if x is not None:
@@ -225,7 +225,7 @@ def parse_props(el):
 
                 # if value has not been parsed then parse it
                 if p_value is None:
-                    p_value = text_type(parse_property.text(el).strip())
+                    p_value = text_type(parse_property.text(el, base_url=self.__url__))
 
 
                 if root_class_names:
@@ -282,7 +282,7 @@ def parse_props(el):
 
                 # if value has not been parsed then parse it
                 if e_value is None:
-                    e_value = parse_property.embedded(el)
+                    e_value = parse_property.embedded(el, base_url=self.__url__)
 
                 if root_class_names:
                     prop_value.append(handle_microformat(

From 2fc6e94a20f99e3f8e1bb4d70ccc11e7abd07cd1 Mon Sep 17 00:00:00 2001
From: Kartik Prabhu <me@kartikprabhu.com>
Date: Sat, 24 Feb 2018 17:58:56 -0500
Subject: [PATCH 04/11] use copy instead of deepcopy

---
 mf2py/dom_helpers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mf2py/dom_helpers.py b/mf2py/dom_helpers.py
index f94b924..1d1c588 100644
--- a/mf2py/dom_helpers.py
+++ b/mf2py/dom_helpers.py
@@ -16,7 +16,7 @@ def get_textContent(el, replace_img=False, base_url=''):
     """
 
     # copy el to avoid making direct changes
-    el_copy = copy.deepcopy(el)
+    el_copy = copy.copy(el)
 
     # drop all <style> and <script> elements
     drops = el_copy.find_all(['style', 'script'])

From cad81e8e678e31e5ca5c4f28b9f1f608d029c38a Mon Sep 17 00:00:00 2001
From: Kartik Prabhu <me@kartikprabhu.com>
Date: Sat, 24 Feb 2018 20:31:19 -0500
Subject: [PATCH 05/11] fix #46 allow balnk attributes whiel parsing properties

---
 mf2py/parse_property.py | 41 ++++++++++++++++++++++++++---------------
 1 file changed, 26 insertions(+), 15 deletions(-)

diff --git a/mf2py/parse_property.py b/mf2py/parse_property.py
index 5eb2329..993c499 100644
--- a/mf2py/parse_property.py
+++ b/mf2py/parse_property.py
@@ -45,10 +45,13 @@ def text(el, base_url=''):
     if value_els:
         return ''.join(get_vcp_value(el) for el in value_els)
 
-    prop_value = get_attr(el, "title", check_name=("abbr", "link"))\
-        or get_attr(el, "value", check_name=("data", "input"))\
-        or get_attr(el, "alt", check_name=("img", "area"))\
-        or get_textContent(el, replace_img=True, base_url=base_url)
+    prop_value = get_attr(el, "title", check_name=("abbr", "link"))
+    if prop_value is None:
+        prop_value = get_attr(el, "value", check_name=("data", "input"))
+    if prop_value is None:
+        prop_value = get_attr(el, "alt", check_name=("img", "area"))
+    if prop_value is None:
+        prop_value = get_textContent(el, replace_img=True, base_url=base_url)
 
     return prop_value
 
@@ -56,10 +59,13 @@ def text(el, base_url=''):
 def url(el, base_url=''):
     """Process u-* properties"""
 
-    prop_value = get_attr(el, "href", check_name=("a", "area", "link"))\
-        or get_attr(el, "src", check_name=("img", "audio", "video", "source"))\
-        or get_attr(el, "poster", check_name="video")\
-        or get_attr(el, "data", check_name="object")
+    prop_value = get_attr(el, "href", check_name=("a", "area", "link"))
+    if prop_value is None:
+        props_value = get_attr(el, "src", check_name=("img", "audio", "video", "source"))
+    if prop_value is None:
+        prop_value = get_attr(el, "poster", check_name="video")
+    if prop_value is None:
+        prop_value = get_attr(el, "data", check_name="object")
 
     if prop_value is not None:
         return urljoin(base_url, prop_value)
@@ -69,9 +75,11 @@ def url(el, base_url=''):
         return urljoin(base_url, ''.join(get_vcp_value(el)
                        for el in value_els))
 
-    prop_value = get_attr(el, "title", check_name="abbr")\
-        or get_attr(el, "value", check_name=("data", "input"))\
-        or get_textContent(el)
+    prop_value = get_attr(el, "title", check_name="abbr")
+    if prop_value is None:
+        prop_value = get_attr(el, "value", check_name=("data", "input"))
+    if prop_value is None:
+        prop_value = get_textContent(el)
 
     return prop_value
 
@@ -164,10 +172,13 @@ def try_normalize(dtstr, match=None):
 
         return try_normalize(date_time_value), date_part
 
-    prop_value = get_attr(el, "datetime", check_name=("time", "ins", "del"))\
-        or get_attr(el, "title", check_name="abbr")\
-        or get_attr(el, "value", check_name=("data", "input"))\
-        or get_textContent(el) 
+    prop_value = get_attr(el, "datetime", check_name=("time", "ins", "del"))
+    if prop_value is None:
+        prop_value = get_attr(el, "title", check_name="abbr")
+    if prop_value is None:
+        prop_value = get_attr(el, "value", check_name=("data", "input"))
+    if prop_value is None:
+        prop_value = get_textContent(el) 
 
     # if this is just a time, augment with default date
     match = re.match(TIME_RE + '$', prop_value)

From 0cde28de363d769bff39aef3a9b6d48b9d9a5663 Mon Sep 17 00:00:00 2001
From: Kartik Prabhu <me@kartikprabhu.com>
Date: Mon, 26 Feb 2018 12:07:25 -0500
Subject: [PATCH 06/11] check doc not self.__doc__ before requesting data

---
 mf2py/parser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mf2py/parser.py b/mf2py/parser.py
index dd88689..7576da0 100644
--- a/mf2py/parser.py
+++ b/mf2py/parser.py
@@ -75,7 +75,7 @@ def __init__(self, doc=None, url=None, html_parser='html5lib'):
         if url is not None:
             self.__url__ = url
 
-            if self.__doc__ is None:
+            if doc is None:
                 data = requests.get(self.__url__, headers={
                     'User-Agent': self.useragent,
                 })

From f4a816e17b9b0660b2d99f5655d8d0cd91aad04d Mon Sep 17 00:00:00 2001
From: Kartik Prabhu <me@kartikprabhu.com>
Date: Thu, 1 Mar 2018 13:23:33 -0500
Subject: [PATCH 07/11] use correct variable name prop_value to parse for u-*

---
 mf2py/parse_property.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mf2py/parse_property.py b/mf2py/parse_property.py
index 993c499..00b51a2 100644
--- a/mf2py/parse_property.py
+++ b/mf2py/parse_property.py
@@ -61,7 +61,7 @@ def url(el, base_url=''):
 
     prop_value = get_attr(el, "href", check_name=("a", "area", "link"))
     if prop_value is None:
-        props_value = get_attr(el, "src", check_name=("img", "audio", "video", "source"))
+        prop_value = get_attr(el, "src", check_name=("img", "audio", "video", "source"))
     if prop_value is None:
         prop_value = get_attr(el, "poster", check_name="video")
     if prop_value is None:

From d91f13ab7af0c20b4acba442b0b1b607e961961b Mon Sep 17 00:00:00 2001
From: Kartik Prabhu <me@kartikprabhu.com>
Date: Sat, 3 Mar 2018 20:16:25 -0500
Subject: [PATCH 08/11] update embedded content value test. add .eggs\ to git
 ignore

---
 .gitignore          | 1 +
 test/test_parser.py | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 50ba3dd..891dec0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,6 +2,7 @@
 *.swp
 *.pyc
 .idea/
+.eggs/
 build/
 dist/
 mf2py.egg-info/
diff --git a/test/test_parser.py b/test/test_parser.py
index 1244ea3..31798f7 100644
--- a/test/test_parser.py
+++ b/test/test_parser.py
@@ -228,7 +228,7 @@ def test_embedded_parsing():
         '   <p>Blah.</p>\n   <p>Blah blah blah.</p>\n  ')
     assert_equal(
         result["items"][0]["properties"]["content"][0]["value"],
-        '\n   Blah blah blah blah blah.\n   Blah.\n   Blah blah blah.\n  ')
+        'Blah blah blah blah blah.\n   Blah.\n   Blah blah blah.')
 
 
 def test_backcompat():

From 550041de1d533585f2464eaa41e1776840b5c3fc Mon Sep 17 00:00:00 2001
From: Kartik Prabhu <me@kartikprabhu.com>
Date: Sun, 4 Mar 2018 10:50:30 -0500
Subject: [PATCH 09/11] strip leading and trailing whitespace from e-*[html].
 update tests

---
 mf2py/parse_property.py | 2 +-
 test/test_parser.py     | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/mf2py/parse_property.py b/mf2py/parse_property.py
index 00b51a2..22f7dfd 100644
--- a/mf2py/parse_property.py
+++ b/mf2py/parse_property.py
@@ -195,6 +195,6 @@ def try_normalize(dtstr, match=None):
 def embedded(el, base_url=''):
     """Process e-* properties"""
     return {
-        'html': el.decode_contents(),    # secret bs4 method to get innerHTML
+        'html': el.decode_contents().strip(),    # secret bs4 method to get innerHTML
         'value': get_textContent(el, replace_img=True, base_url=base_url)
     }
diff --git a/test/test_parser.py b/test/test_parser.py
index 31798f7..617a730 100644
--- a/test/test_parser.py
+++ b/test/test_parser.py
@@ -224,8 +224,8 @@ def test_embedded_parsing():
     result = parse_fixture("embedded.html")
     assert_equal(
         result["items"][0]["properties"]["content"][0]["html"],
-        '\n   <p>Blah blah blah blah blah.</p>\n' +
-        '   <p>Blah.</p>\n   <p>Blah blah blah.</p>\n  ')
+        '<p>Blah blah blah blah blah.</p>\n' +
+        '   <p>Blah.</p>\n   <p>Blah blah blah.</p>')
     assert_equal(
         result["items"][0]["properties"]["content"][0]["value"],
         'Blah blah blah blah blah.\n   Blah.\n   Blah blah blah.')

From bf10d47ce9c406444f72b7c624f67372e74b3ec4 Mon Sep 17 00:00:00 2001
From: Kartik Prabhu <me@kartikprabhu.com>
Date: Sun, 4 Mar 2018 12:08:44 -0500
Subject: [PATCH 10/11] update requirements on BS4 and html5lib

---
 requirements.txt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 4a1ff4a..4a0caff 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,8 +1,8 @@
-html5lib==0.999
+html5lib==1.0.1
 nose==1.3.0
 mock==1.3.0
-requests==2.0.1
-BeautifulSoup4==4.3.2
+requests==2.18.4
+BeautifulSoup4==4.6.0
 Flask==0.10.1
 gunicorn==19.1.1
 -e .

From de04a20ec7684bf19b46c461b60d514026528334 Mon Sep 17 00:00:00 2001
From: Kartik Prabhu <me@kartikprabhu.com>
Date: Sun, 4 Mar 2018 12:46:21 -0500
Subject: [PATCH 11/11] make basetring to str for python3 usage

---
 mf2py/dom_helpers.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/mf2py/dom_helpers.py b/mf2py/dom_helpers.py
index 1d1c588..62273c3 100644
--- a/mf2py/dom_helpers.py
+++ b/mf2py/dom_helpers.py
@@ -10,6 +10,7 @@
     from urllib.parse import urljoin
     text_type = str
     binary_type = bytes
+    basestring = str
 
 def get_textContent(el, replace_img=False, base_url=''):
     """ Get the text content of an element, replacing images by alt or src