From 3a8102d56f5f22171801d48c85084429a1235607 Mon Sep 17 00:00:00 2001 From: Jonathan Cameron Date: Sat, 10 Feb 2018 23:11:37 -0800 Subject: [PATCH 1/5] Added fixes for QIF files with different date order and embeded thousands separateros --- qifparse/parser.py | 37 ++++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/qifparse/parser.py b/qifparse/parser.py index 9ccebe7..0acd99b 100644 --- a/qifparse/parser.py +++ b/qifparse/parser.py @@ -13,10 +13,18 @@ Qif, ) + +def convertFloat(num_str): + """Convert float, possibly with embedded thousands separators""" + num_str = num_str.replace(QifParser._thousands_separator, '') + return Decimal(num_str) + + NON_INVST_ACCOUNT_TYPES = [ '!Type:Cash', '!Type:Bank', '!Type:Ccard', + '!Type:CCard', # Some Quicken exports '!Type:Oth A', '!Type:Oth L', '!Type:Invoice', # Quicken for business only @@ -29,8 +37,15 @@ class QifParserException(Exception): class QifParser(object): + _months_first = False + _thousands_separator = ',' + @classmethod - def parse(cls_, file_handle, date_format=None): + def parse(cls_, file_handle, date_format=None, months_first=None, thousands_separator=None): + if months_first is not None: + cls_._months_first = months_first + if thousands_separator is not None: + cls_._thousands_separator = thousands_separator if isinstance(file_handle, type('')): raise RuntimeError( six.u("parse() takes in a file handle, not a string")) @@ -54,6 +69,7 @@ def parse(cls_, file_handle, date_format=None): if not chunk: continue first_line = chunk.split('\n')[0] + first_line = first_line.rstrip() if first_line == '!Type:Cat': last_type = 'category' elif first_line == '!Account': @@ -232,7 +248,7 @@ def parseTransaction(cls_, chunk, date_format=None): elif line[0] == 'N': curItem.num = line[1:] elif line[0] == 'T': - curItem.amount = Decimal(line[1:]) + curItem.amount = convertFloat(line[1:]) elif line[0] == 'C': curItem.cleared = line[1:] elif line[0] == 'P': @@ -281,7 +297,7 @@ def parseTransaction(cls_, chunk, date_format=None): split.address.append(line[1:]) elif line[0] == '$': split = curItem.splits[-1] - split.amount = Decimal(line[1:]) + split.amount = convertFloat(line[1:-1]) else: # don't recognise this line; ignore it print ("Skipping unknown line:\n" + str(line)) @@ -348,5 +364,16 @@ def parseQifDateTime(cls_, qdate): C = "20" else: C = "19" - iso_date = C + qdate[6:8] + "-" + qdate[3:5] + "-" + qdate[0:2] - return datetime.strptime(iso_date, '%Y-%m-%d') + year = C + qdate[6:8] + if cls_._months_first: + month = qdate[0:2] + day = qdate[3:5] + else: + month = qdate[3:5] + day = qdate[0:2] + iso_date = year + "-" + month + "-" + day + try: + dtime = datetime.strptime(iso_date, '%Y-%m-%d') + except: + raise ValueError("ERROR in time format QIF(%s) ISO(%s)" % (qdate, iso_date)) + return dtime From 8990d49de9d3f51d7a7c287448e9cbdfb6e79d24 Mon Sep 17 00:00:00 2001 From: Jonathan Cameron Date: Sun, 11 Feb 2018 22:27:35 -0800 Subject: [PATCH 2/5] Fixes to get regtest working --- qifparse/parser.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/qifparse/parser.py b/qifparse/parser.py index 0acd99b..86e4840 100644 --- a/qifparse/parser.py +++ b/qifparse/parser.py @@ -225,7 +225,7 @@ def parseMemorizedTransaction(cls_, chunk, date_format=None): split.address.append(line[1:]) elif line[0] == '$': split = curItem.splits[-1] - split.amount = Decimal(line[1:]) + split.amount = convertFloat(line[1:]) else: # don't recognise this line; ignore it print ("Skipping unknown line:\n" + str(line)) @@ -297,7 +297,8 @@ def parseTransaction(cls_, chunk, date_format=None): split.address.append(line[1:]) elif line[0] == '$': split = curItem.splits[-1] - split.amount = convertFloat(line[1:-1]) + # split.amount = convertFloat(line[1:-1]) + split.amount = "%.2f" % convertFloat(line[1:-1]) else: # don't recognise this line; ignore it print ("Skipping unknown line:\n" + str(line)) @@ -318,7 +319,7 @@ def parseInvestment(cls_, chunk, date_format=None): elif line[0] == 'D': curItem.date = cls_.parseQifDateTime(line[1:]) elif line[0] == 'T': - curItem.amount = Decimal(line[1:]) + curItem.amount = convertFloat(line[1:]) elif line[0] == 'N': curItem.action = line[1:] elif line[0] == 'Y': From 710e32b8bc25c94e17524c080f61f2b895a3903f Mon Sep 17 00:00:00 2001 From: Jonathan Cameron Date: Mon, 12 Feb 2018 07:10:57 -0800 Subject: [PATCH 3/5] Better fix for formatting problem (in regtests) --- qifparse/parser.py | 3 +-- qifparse/qif.py | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/qifparse/parser.py b/qifparse/parser.py index 86e4840..f4f552e 100644 --- a/qifparse/parser.py +++ b/qifparse/parser.py @@ -297,8 +297,7 @@ def parseTransaction(cls_, chunk, date_format=None): split.address.append(line[1:]) elif line[0] == '$': split = curItem.splits[-1] - # split.amount = convertFloat(line[1:-1]) - split.amount = "%.2f" % convertFloat(line[1:-1]) + split.amount = convertFloat(line[1:-1]) else: # don't recognise this line; ignore it print ("Skipping unknown line:\n" + str(line)) diff --git a/qifparse/qif.py b/qifparse/qif.py index 2317f82..c765c7f 100644 --- a/qifparse/qif.py +++ b/qifparse/qif.py @@ -185,7 +185,7 @@ class Transaction(BaseEntry): _fields = [ Field('date', 'datetime', 'D', required=True, default=datetime.now()), Field('num', 'string', 'N'), - Field('amount', 'decimal', 'T', required=True), + Field('amount', 'decimal', 'T', required=True, custom_print_format='%s%.2f'), Field('cleared', 'string', 'C'), Field('payee', 'string', 'P'), Field('memo', 'string', 'M'), @@ -240,7 +240,7 @@ class AmountSplit(BaseEntry): _fields = [ Field('category', 'string', 'S'), Field('to_account', 'reference', 'S'), - Field('amount', 'decimal', '$'), + Field('amount', 'decimal', '$', custom_print_format='%s%.2f'), Field('percent', 'string', '%'), Field('address', 'multilinestring', 'A'), Field('memo', 'string', 'M'), From f56da7b5c4c4400af08e11ffcc07eeb587c00364 Mon Sep 17 00:00:00 2001 From: Jonathan Cameron Date: Mon, 12 Feb 2018 12:21:36 -0800 Subject: [PATCH 4/5] Update to improve handling of currency formats --- qifparse/parser.py | 9 ++++++++- qifparse/qif.py | 24 +++++++++++++++++++++--- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/qifparse/parser.py b/qifparse/parser.py index f4f552e..f36ffd0 100644 --- a/qifparse/parser.py +++ b/qifparse/parser.py @@ -41,11 +41,16 @@ class QifParser(object): _thousands_separator = ',' @classmethod - def parse(cls_, file_handle, date_format=None, months_first=None, thousands_separator=None): + def parse(cls_, file_handle, date_format=None, + months_first=None, + thousands_separator=None, + currency_num_fractional_digits=None): if months_first is not None: cls_._months_first = months_first if thousands_separator is not None: cls_._thousands_separator = thousands_separator + if currency_num_fractional_digits is not None: + Qif.set_currency_num_fractional_digits(int(currency_num_fractional_digits)) if isinstance(file_handle, type('')): raise RuntimeError( six.u("parse() takes in a file handle, not a string")) @@ -249,6 +254,8 @@ def parseTransaction(cls_, chunk, date_format=None): curItem.num = line[1:] elif line[0] == 'T': curItem.amount = convertFloat(line[1:]) + elif line[0] == 'U': + curItem.amount_U = convertFloat(line[1:]) elif line[0] == 'C': curItem.cleared = line[1:] elif line[0] == 'P': diff --git a/qifparse/qif.py b/qifparse/qif.py index c765c7f..4e82816 100644 --- a/qifparse/qif.py +++ b/qifparse/qif.py @@ -23,6 +23,9 @@ class Qif(object): + + _currency_format = '%.2f' + def __init__(self): self._accounts = [] self._categories = [] @@ -103,6 +106,17 @@ def get_transactions(self, recursive=False): for acc in self._accounts: tr.extend(acc.transactions) + @classmethod + def set_currency_num_fractional_digits(cls, num): + """Set the number of digits in the fractional part of currency values + (for __str__ output)""" + cls._currency_format = "%%.%df" % int(num) + print(" CUR FORM: ", Qif._currency_format) + + @classmethod + def currency_format(cls): + return cls._currency_format + def __str__(self): res = [] if self._categories: @@ -185,7 +199,10 @@ class Transaction(BaseEntry): _fields = [ Field('date', 'datetime', 'D', required=True, default=datetime.now()), Field('num', 'string', 'N'), - Field('amount', 'decimal', 'T', required=True, custom_print_format='%s%.2f'), + Field('amount', 'decimal', 'T', required=True, + custom_print_format='%s' + Qif.currency_format()), + Field('amount_U', 'decimal', 'U', + custom_print_format='%s' + Qif.currency_format()), Field('cleared', 'string', 'C'), Field('payee', 'string', 'P'), Field('memo', 'string', 'M'), @@ -240,7 +257,8 @@ class AmountSplit(BaseEntry): _fields = [ Field('category', 'string', 'S'), Field('to_account', 'reference', 'S'), - Field('amount', 'decimal', '$', custom_print_format='%s%.2f'), + Field('amount', 'decimal', '$', + custom_print_format='%s' + Qif.currency_format()), Field('percent', 'string', '%'), Field('address', 'multilinestring', 'A'), Field('memo', 'string', 'M'), @@ -253,7 +271,7 @@ class Investment(BaseEntry): Field('date', 'datetime', 'D', required=True, default=datetime.now()), Field('action', 'string', 'N'), Field('security', 'string', 'Y'), - Field('price', 'decimal', 'I', custom_print_format='%s%.3f'), + Field('price', 'decimal', 'I', custom_print_format='%s%.3f'), # Why 3? Field('quantity', 'decimal', 'Q', custom_print_format='%s%.3f'), Field('cleared', 'string', 'C'), Field('amount', 'decimal', 'T'), From b3b0448cc5f4aabb68ea02faec085b7d4b6ee85b Mon Sep 17 00:00:00 2001 From: Jonathan Cameron Date: Mon, 12 Feb 2018 13:33:42 -0800 Subject: [PATCH 5/5] Fixed another month-order issue --- qifparse/parser.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/qifparse/parser.py b/qifparse/parser.py index f36ffd0..751886e 100644 --- a/qifparse/parser.py +++ b/qifparse/parser.py @@ -74,7 +74,7 @@ def parse(cls_, file_handle, date_format=None, if not chunk: continue first_line = chunk.split('\n')[0] - first_line = first_line.rstrip() + first_line = first_line.strip() if first_line == '!Type:Cat': last_type = 'category' elif first_line == '!Account': @@ -364,9 +364,22 @@ def parseQifDateTime(cls_, qdate): for i in range(len(qdate)): if qdate[i] == " ": qdate = qdate[:i] + "0" + qdate[i+1:] + if len(qdate) == 10: # new form with YYYY date - iso_date = qdate[6:10] + "-" + qdate[3:5] + "-" + qdate[0:2] - return datetime.strptime(iso_date, '%Y-%m-%d') + year = qdate[6:10] + if cls_._months_first: + month = qdate[0:2] + day = qdate[3:5] + else: + month = qdate[3:5] + day = qdate[0:2] + iso_date = year + "-" + month + "-" + day + try: + dtime = datetime.strptime(iso_date, '%Y-%m-%d') + except: + raise ValueError("ERROR in time format QIF(%s) ISO(%s)" % (qdate, iso_date)) + return dtime + if qdate[5] == "'": C = "20" else: