Reduce required bank import

byro_shackspace/management/commands/import_shackbureau.py

@@ -3,6 +3,7 @@ from datetime import datetime, time, timedelta
 from decimal import Decimal
 
 import pytz
+from byro_shackspace.models import ShackProfile
 from django.core.management.base import BaseCommand
 from django.db import transaction
 from django.utils.dateparse import parse_date
@@ -12,7 +13,6 @@ from byro.bookkeeping.models import (
     TransactionChannel, VirtualTransaction,
 )
 from byro.members.models import Member, Membership
-from byro_shackspace.models import ShackProfile
 
 TIMEZONE = pytz.timezone('Europe/Berlin')
 

byro_shackspace/utils.py

@@ -1,163 +1,64 @@
 import csv
 import re
+from datetime import date, datetime, timedelta
 from decimal import Decimal
-from datetime import datetime, date, timedelta
+
+from django.db import transaction
+from django.utils.timezone import now
+
+from byro.bookkeeping.models import RealTransaction, TransactionChannel
+from byro.members.models import Member
 
 
-class TransactionLogProcessor:
+@transaction.atomic
-    def __init__(self):
+def process_bank_csv(self, source):
-        self.debitors = Debitor.objects.all().values('pk', 'record_token', 'record_token_line_2')
+    reader = csv.DictReader(source.source_file.name, encoding='iso-8859-1', delimiter=';', quotechar='"')
+    booking_timestamp = now()
 
-        def build_regex(record_token):
-            record_token = [re.escape(element) for element in record_token.lower().split()]
-            regex = r".*" + r'\s*'.join(record_token) + ".*"
-            return regex
-
-        for i in range(len(self.debitors)):
-            self.debitors[i]['regex'] = build_regex(self.debitors[i]['record_token'])
-            self.debitors[i]['regex_line_2'] = build_regex(self.debitors[i]['record_token_line_2'])
-
-    def process(self, banktransaction):
-        if banktransaction.data_type == 'bank_csv':
-            self.process_bank_csv(banktransaction)
-        else:
-            self.process_accountant_csv(banktransaction)
-
-    def process_accountant_csv(self, banktransaction):
-        banktransaction.status = 'wip'
-        banktransaction.save()
-        reader = csv.reader(open(banktransaction.data_file.file.name, encoding='iso-8859-1'),
-                            delimiter=";", quotechar='"')
-        reader.__next__()  # first line is meta of accountant
-        header = reader.__next__()  # second line is header
     for line in reader:
         if not line:
             continue
-            d = dict(zip(header, line))
-            member = None
-            uid = None
-            error = None
-            if 'Buchungstext' in d:
-                members = Member.objects.filter(surname__iexact=d.get('Buchungstext'))
-                if members.count() == 1:
-                    member = members.first()
-                    uid = member.member_id
-            reference = d.get('Buchungstext')
-            haben = Decimal(d.get('Umsatz Haben').replace(',', '.') or 0)
-            soll = Decimal(d.get('Umsatz Soll').replace(',', '.') or 0)
-            amount = haben - soll
-            BankTransactionLog.objects.create(
-                upload=banktransaction,
-                reference=reference,
-                member=member,
-                error=error, score=0,
-                amount=amount,
-                booking_date=datetime.strptime(d.get('Datum'), '%d.%m.%Y').date(),
-                is_matched=bool(uid),
-                is_resolved=bool(uid),
-                created_by=banktransaction.created_by
-            )
-            if member:
-                defaults = {
-                    'transaction_type': 'membership fee',
-                    'amount': amount,
-                    'created_by': banktransaction.created_by,
-                    'payment_reference': reference
-                }
-                due_date = datetime.strptime(d.get('Datum'), '%d.%m.%Y').date()
-                transation_hash = hashlib.sha256((';'.join(line)).encode('utf-8')).hexdigest()
-                AccountTransaction.objects.update_or_create(
-                    booking_type='deposit',
-                    member=member,
-                    due_date=due_date,
-                    transaction_hash=transation_hash,
-                    defaults=defaults)
-        banktransaction.status = 'done'
-        banktransaction.save()
-
-    def process_bank_csv(self, banktransaction):
-        banktransaction.status = 'wip'
-        banktransaction.save()
-        reader = csv.reader(open(banktransaction.data_file.file.name, encoding='iso-8859-1'),
-                            delimiter=";", quotechar='"')
-        header = reader.__next__()
-        for line in reader:
-            if not line:
-                continue
-            d = dict(zip(header, line))
         reference = ''
-            for key in sorted(header):
+        for key in line.keys():
             if key.startswith('VWZ'):
                 reference += d[key] + ' '
 
-            uid, score = self.reference_parser(reference)
+        RealTransaction.objects.create(
+            channel=TransactionChannel.BANK,
+            booking_datetime=booking_timestamp,
+            value_datetime=datetime.strptime(line.get('Buchungstag'), '%d.%m.%Y'),
+            amount=Decimal(line.get('Betrag').replace('.', '').replace(',', '.')),
+            purpose=reference,
+            originator=line.get('Auftraggeber/Empfänger'),
+            importer='shack_bank_csv_importer',
+            source=source,
+            data=line,
+        )
+
+
+def match_transaction(transaction):
+    uid, score = reference_parser(transaction.reference)
     member = None
-            debitor = self.get_debitor_by_record_token(reference)
     error = None
     try:
         if uid:
             member = Member.objects.get(member_id=uid)
     except Member.DoesNotExist:
         error = "Member does not exist"
-            BankTransactionLog.objects.create(
-                upload=banktransaction,
-                reference=reference,
-                member=member,
-                debitor=debitor,
-                error=error, score=score,
-                amount=Decimal(d.get('Betrag').replace('.', '').replace(',', '.')),
-                booking_date=datetime.strptime(d.get('Buchungstag'), '%d.%m.%Y').date(),
-                transaction_owner=d.get('Auftraggeber/Empfänger'),
-                is_matched=bool(uid) or bool(debitor),
-                is_resolved=bool(uid) or bool(debitor),
-                created_by=banktransaction.created_by
-            )
-            if member:
-                defaults = {
-                    'transaction_type': 'membership fee',
-                    'amount': Decimal(d.get('Betrag').replace('.', '').replace(',', '.')),
-                    'created_by': banktransaction.created_by,
-                    'payment_reference': reference
-                }
-                due_date = datetime.strptime(d.get('Buchungstag'), '%d.%m.%Y').date()
-                transation_hash = hashlib.sha256((';'.join(line)).encode('utf-8')).hexdigest()
-                AccountTransaction.objects.update_or_create(
-                    booking_type='deposit',
-                    member=member,
-                    due_date=due_date,
-                    transaction_hash=transation_hash,
-                    defaults=defaults)
-            elif debitor:
-                defaults = {
-                    'amount': Decimal(d.get('Betrag').replace('.', '').replace(',', '.')),
-                    'created_by': banktransaction.created_by,
-                    'payment_reference': reference
-                }
-                due_date = datetime.strptime(d.get('Buchungstag'), '%d.%m.%Y').date()
-                transation_hash = hashlib.sha256((';'.join(line)).encode('utf-8')).hexdigest()
-                DistrictcourtAccountTransaction.objects.update_or_create(
-                    booking_type='deposit',
-                    debitor=debitor,
-                    due_date=due_date,
-                    transaction_hash=transation_hash,
-                    defaults=defaults)
 
-        banktransaction.status = 'done'
+def reference_parser(self, reference):
-        banktransaction.save()
-
-    def reference_parser(self, reference):
     reference = reference.lower()
 
     regexes = (
         r'.*mitgliedsbeitrag\s+id\s+(?P<ID>\d{1,4})\s.*',
         r'.*id\s+(?P<ID>\d{1,4})\smitgliedsbeitrag.*',
-            r'.*id\s+(?P<ID>\d{1,4})\s.*',
+        # r'.*id\s+(?P<ID>\d{1,4})\s.*',
         r'.*mitgliedsbeitrag.*id\s+(?P<ID>\d{1,4})\s.*',
         r'.*mitgliedsbeitrag\s+(?P<ID>\d{1,4})\s.*',
         r'.*beitrag\s+mitglied\s+(?P<ID>\d{1,4})\s.*',
         r'.*mitgliedsbeitrag.*\s+(?P<ID>\d{1,4})[^\d].*',
-            r'.*id(?P<ID>\d{1,4})\s+zr\d+.*',
+        # r'.*id(?P<ID>\d{1,4})\s+zr\d+.*',
-            r'.*id\s+(?P<ID>\d{1,4}),\s+zr\s+\d+.*',
+        # r'.*id\s+(?P<ID>\d{1,4}),\s+zr\s+\d+.*',
         r'.*mitgliedsbeitrag\s+id[.:-_](?P<ID>\d{1,4})\s.*',
     )
 
@@ -168,7 +69,7 @@ class TransactionLogProcessor:
 
     return (False, 99)
 
-    def get_debitor_by_record_token(self, reference):
+def get_debitor_by_record_token(self, reference):
     reference = "".join(reference.lower().split())
     for debitor in self.debitors:
         if re.match(debitor['regex'], reference):