X-Git-Url: http://wagnertech.de/gitweb/gitweb.cgi/mfinanz.git/blobdiff_plain/5edee2313bde03fb9b825905a36647569b504199..b3089c09460c250bfab75cf84c34f4ea958dc2e1:/SL/DATEV.pm
diff --git a/SL/DATEV.pm b/SL/DATEV.pm
index cd04053ba..3766c1b60 100644
--- a/SL/DATEV.pm
+++ b/SL/DATEV.pm
@@ -1,5 +1,5 @@
 #=====================================================================
-# Lx-Office ERP
+# kivitendo ERP
 # Copyright (c) 2004
 #
 #  Author: Philip Reetz
@@ -18,31 +18,92 @@
 # GNU General Public License for more details.
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA 02110-1335, USA.
 #======================================================================
 #
 # Datev export module
 #======================================================================
 
-package DATEV;
+package SL::DATEV;
 
 use utf8;
 use strict;
 
 use SL::DBUtils;
 use SL::DATEV::KNEFile;
-use SL::Taxkeys;
+use SL::DATEV::CSV;
+use SL::DB;
+use SL::HTML::Util ();
+use SL::Iconv;
+use SL::Locale::String qw(t8);
+use SL::VATIDNr;
 
 use Data::Dumper;
+use DateTime;
+use Exporter qw(import);
 use File::Path;
-use List::Util qw(max);
+use IO::File;
+use List::MoreUtils qw(any);
+use List::Util qw(min max sum);
+use List::UtilsBy qw(partition_by sort_by);
+use Text::CSV_XS;
 use Time::HiRes qw(gettimeofday);
 
+{
+  my $i = 0;
+  use constant {
+    DATEV_ET_BUCHUNGEN => $i++,
+    DATEV_ET_STAMM     => $i++,
+    DATEV_ET_CSV       => $i++,
+
+    DATEV_FORMAT_KNE   => $i++,
+    DATEV_FORMAT_OBE   => $i++,
+    DATEV_FORMAT_CSV   => $i++,
+  };
+}
+
+my @export_constants = qw(DATEV_ET_BUCHUNGEN DATEV_ET_STAMM DATEV_ET_CSV DATEV_FORMAT_KNE DATEV_FORMAT_OBE DATEV_FORMAT_CSV);
+our @EXPORT_OK = (@export_constants);
+our %EXPORT_TAGS = (CONSTANTS => [ @export_constants ]);
+
+
+sub new {
+  my $class = shift;
+  my %data  = @_;
+
+  my $obj = bless {}, $class;
+
+  $obj->$_($data{$_}) for keys %data;
+
+  $obj;
+}
+
+sub exporttype {
+  my $self = shift;
+  $self->{exporttype} = $_[0] if @_;
+  return $self->{exporttype};
+}
+
+sub has_exporttype {
+  defined $_[0]->{exporttype};
+}
+
+sub format {
+  my $self = shift;
+  $self->{format} = $_[0] if @_;
+  return $self->{format};
+}
+
+sub has_format {
+  defined $_[0]->{format};
+}
+
 sub _get_export_path {
   $main::lxdebug->enter_sub();
 
   my ($a, $b) = gettimeofday();
-  my $path    = get_path_for_download_token("${a}-${b}-${$}");
+  my $path    = _get_path_for_download_token("${a}-${b}-${$}");
 
   mkpath($path) unless (-d $path);
 
@@ -51,14 +112,14 @@ sub _get_export_path {
   return $path;
 }
 
-sub get_path_for_download_token {
+sub _get_path_for_download_token {
   $main::lxdebug->enter_sub();
 
-  my $token = shift;
+  my $token = shift || '';
   my $path;
 
   if ($token =~ m|^(\d+)-(\d+)-(\d+)$|) {
-    $path = $::lx_office_conf{paths}->{userspath} . "/datev-export-${1}-${2}-${3}";
+    $path = $::lx_office_conf{paths}->{userspath} . "/datev-export-${1}-${2}-${3}/";
   }
 
   $main::lxdebug->leave_sub();
@@ -66,7 +127,7 @@ sub get_path_for_download_token {
   return $path;
 }
 
-sub get_download_token_for_path {
+sub _get_download_token_for_path {
   $main::lxdebug->enter_sub();
 
   my $path = shift;
@@ -81,11 +142,142 @@ sub get_download_token_for_path {
   return $token;
 }
 
+sub download_token {
+  my $self = shift;
+  $self->{download_token} = $_[0] if @_;
+  return $self->{download_token} ||= _get_download_token_for_path($self->export_path);
+}
+
+sub export_path {
+  my ($self) = @_;
+
+  return  $self->{export_path} ||= _get_path_for_download_token($self->{download_token}) || _get_export_path();
+}
+
+sub add_filenames {
+  my $self = shift;
+  push @{ $self->{filenames} ||= [] }, @_;
+}
+
+sub filenames {
+  return @{ $_[0]{filenames} || [] };
+}
+
+sub add_error {
+  my $self = shift;
+  push @{ $self->{errors} ||= [] }, @_;
+}
+
+sub errors {
+  return @{ $_[0]{errors} || [] };
+}
+
+sub add_net_gross_differences {
+  my $self = shift;
+  push @{ $self->{net_gross_differences} ||= [] }, @_;
+}
+
+sub net_gross_differences {
+  return @{ $_[0]{net_gross_differences} || [] };
+}
+
+sub sum_net_gross_differences {
+  return sum $_[0]->net_gross_differences;
+}
+
+sub from {
+ my $self = shift;
+
+ if (@_) {
+   $self->{from} = $_[0];
+ }
+
+ return $self->{from};
+}
+
+sub to {
+ my $self = shift;
+
+ if (@_) {
+   $self->{to} = $_[0];
+ }
+
+ return $self->{to};
+}
+
+sub trans_id {
+  my $self = shift;
+
+  if (@_) {
+    $self->{trans_id} = $_[0];
+  }
+
+  die "illegal trans_id passed for DATEV export: " . $self->{trans_id} . "\n" unless $self->{trans_id} =~ m/^\d+$/;
+
+  return $self->{trans_id};
+}
+
+sub warnings {
+  my $self = shift;
+
+  if (@_) {
+    $self->{warnings} = [@_];
+  } else {
+   return $self->{warnings};
+  }
+}
+
+sub use_pk {
+ my $self = shift;
+
+ if (@_) {
+   $self->{use_pk} = $_[0];
+ }
+
+ return $self->{use_pk};
+}
+
+sub accnofrom {
+ my $self = shift;
+
+ if (@_) {
+   $self->{accnofrom} = $_[0];
+ }
+
+ return $self->{accnofrom};
+}
+
+sub accnoto {
+ my $self = shift;
+
+ if (@_) {
+   $self->{accnoto} = $_[0];
+ }
+
+ return $self->{accnoto};
+}
+
+
+sub dbh {
+  my $self = shift;
+
+  if (@_) {
+    $self->{dbh} = $_[0];
+    $self->{provided_dbh} = 1;
+  }
+
+  $self->{dbh} ||= SL::DB->client->dbh;
+}
+
+sub provided_dbh {
+  $_[0]{provided_dbh};
+}
+
 sub clean_temporary_directories {
-  $main::lxdebug->enter_sub();
+  $::lxdebug->enter_sub;
 
   foreach my $path (glob($::lx_office_conf{paths}->{userspath} . "/datev-export-*")) {
-    next unless (-d $path);
+    next unless -d $path;
 
     my $mtime = (stat($path))[9];
     next if ((time() - $mtime) < 8 * 60 * 60);
@@ -93,13 +285,13 @@ sub clean_temporary_directories {
     rmtree $path;
   }
 
-  $main::lxdebug->leave_sub();
+  $::lxdebug->leave_sub;
 }
 
 sub _fill {
   $main::lxdebug->enter_sub();
 
-  my $text      = shift;
+  my $text      = shift // '';
   my $field_len = shift;
   my $fill_char = shift;
   my $alignment = shift || 'right';
@@ -120,294 +312,360 @@ sub _fill {
 }
 
 sub get_datev_stamm {
-  $main::lxdebug->enter_sub();
-
-  my ($self, $myconfig, $form) = @_;
-
-  # connect to database
-  my $dbh = $form->dbconnect($myconfig);
+  return $_[0]{stamm} ||= selectfirst_hashref_query($::form, $_[0]->dbh, 'SELECT * FROM datev');
+}
 
-  my $query = qq|SELECT * FROM datev|;
-  my $sth   = $dbh->prepare($query);
-  $sth->execute || $form->dberror($query);
+sub save_datev_stamm {
+  my ($self, $data) = @_;
 
-  my $ref = $sth->fetchrow_hashref("NAME_lc");
+  SL::DB->client->with_transaction(sub {
+    do_query($::form, $self->dbh, 'DELETE FROM datev');
 
-  map { $form->{$_} = $ref->{$_} } keys %$ref;
+    my @columns = qw(beraternr beratername dfvkz mandantennr datentraegernr abrechnungsnr);
 
-  $sth->finish;
-  $dbh->disconnect;
-  $main::lxdebug->leave_sub();
+    my $query = "INSERT INTO datev (" . join(', ', @columns) . ") VALUES (" . join(', ', ('?') x @columns) . ")";
+    do_query($::form, $self->dbh, $query, map { $data->{$_} } @columns);
+    1;
+  }) or do { die SL::DB->client->error };
 }
 
-sub save_datev_stamm {
-  $main::lxdebug->enter_sub();
+sub export {
+  my ($self) = @_;
+  my $result;
 
-  my ($self, $myconfig, $form) = @_;
-
-  # connect to database
-  my $dbh = $form->dbconnect_noauto($myconfig);
-
-  my $query = qq|DELETE FROM datev|;
-  $dbh->do($query) || $form->dberror($query);
-
-  $query = qq|INSERT INTO datev
-              (beraternr, beratername, dfvkz, mandantennr, datentraegernr, abrechnungsnr) VALUES
-              (|
-    . $dbh->quote($form->{beraternr}) . qq|,|
-    . $dbh->quote($form->{beratername}) . qq|,|
-    . $dbh->quote($form->{dfvkz}) . qq|,
-              |
-    . $dbh->quote($form->{mandantennr}) . qq|,|
-    . $dbh->quote($form->{datentraegernr}) . qq|,|
-    . $dbh->quote($form->{abrechnungsnr}) . qq|)|;
-  my $sth = $dbh->prepare($query);
-  $sth->execute || $form->dberror($query);
-  $sth->finish;
+  die 'no format set!' unless $self->has_format;
 
-  $dbh->commit;
-  $dbh->disconnect;
-  $main::lxdebug->leave_sub();
+  if ($self->format == DATEV_FORMAT_CSV) {
+    $result = $self->csv_export;
+  } elsif ($self->format == DATEV_FORMAT_KNE) {
+    $result = $self->kne_export;
+  } elsif ($self->format == DATEV_FORMAT_OBE) {
+    $result = $self->obe_export;
+  } else {
+    die 'unrecognized export format';
+  }
+
+  return $result;
 }
 
 sub kne_export {
-  $main::lxdebug->enter_sub();
-
-  my ($self, $myconfig, $form) = @_;
+  my ($self) = @_;
   my $result;
 
-  if ($form->{exporttype} == 0) {
-    $result = kne_buchungsexport($myconfig, $form);
+  die 'no exporttype set!' unless $self->has_exporttype;
+
+  if ($self->exporttype == DATEV_ET_BUCHUNGEN) {
+    $result = $self->kne_buchungsexport;
+  } elsif ($self->exporttype == DATEV_ET_STAMM) {
+    $result = $self->kne_stammdatenexport;
+  } elsif ($self->exporttype == DATEV_ET_CSV) {
+    $result = $self->csv_export_for_tax_accountant;
   } else {
-    $result = kne_stammdatenexport($myconfig, $form);
+    die 'unrecognized exporttype';
   }
 
-  $main::lxdebug->leave_sub();
-
   return $result;
 }
 
-sub obe_export {
-  $main::lxdebug->enter_sub();
+sub csv_export {
+  my ($self) = @_;
+  my $result;
 
-  my ($self, $myconfig, $form) = @_;
+  die 'no exporttype set!' unless $self->has_exporttype;
 
-  # connect to database
-  my $dbh = $form->dbconnect_noauto($myconfig);
-  $dbh->commit;
-  $dbh->disconnect;
-  $main::lxdebug->leave_sub();
-}
+  if ($self->exporttype == DATEV_ET_BUCHUNGEN) {
 
-sub get_dates {
-  $main::lxdebug->enter_sub();
+    $self->generate_datev_data(from_to => $self->fromto);
+    return if $self->errors;
 
-  my ($zeitraum, $monat, $quartal, $transdatefrom, $transdateto) = @_;
-  my ($fromto, $jahr, $leap);
+    my $datev_csv = SL::DATEV::CSV->new(
+      datev_lines  => $self->generate_datev_lines,
+      from         => $self->from,
+      to           => $self->to,
+      locked       => $self->locked,
+    );
 
-  my $form = $main::form;
 
-  $fromto = "transdate >= ";
+    my $filename = "EXTF_DATEV_kivitendo" . $self->from->ymd() . '-' . $self->to->ymd() . ".csv";
 
-  my @a = localtime;
-  $a[5] += 1900;
-  $jahr = $a[5];
-  if ($zeitraum eq "monat") {
-  SWITCH: {
-      $monat eq "1" && do {
-        $form->{fromdate} = "1.1.$jahr";
-        $form->{todate}   = "31.1.$jahr";
-        last SWITCH;
-      };
-      $monat eq "2" && do {
-        $form->{fromdate} = "1.2.$jahr";
+    my $csv = Text::CSV_XS->new({
+                binary       => 1,
+                sep_char     => ";",
+                always_quote => 1,
+                eol          => "\r\n",
+              }) or die "Cannot use CSV: ".Text::CSV_XS->error_diag();
 
-        #this works from 1901 to 2099, 1900 and 2100 fail.
-        $leap = ($jahr % 4 == 0) ? "29" : "28";
-        $form->{todate} = "$leap.2.$jahr";
-        last SWITCH;
-      };
-      $monat eq "3" && do {
-        $form->{fromdate} = "1.3.$jahr";
-        $form->{todate}   = "31.3.$jahr";
-        last SWITCH;
-      };
-      $monat eq "4" && do {
-        $form->{fromdate} = "1.4.$jahr";
-        $form->{todate}   = "30.4.$jahr";
-        last SWITCH;
-      };
-      $monat eq "5" && do {
-        $form->{fromdate} = "1.5.$jahr";
-        $form->{todate}   = "31.5.$jahr";
-        last SWITCH;
-      };
-      $monat eq "6" && do {
-        $form->{fromdate} = "1.6.$jahr";
-        $form->{todate}   = "30.6.$jahr";
-        last SWITCH;
-      };
-      $monat eq "7" && do {
-        $form->{fromdate} = "1.7.$jahr";
-        $form->{todate}   = "31.7.$jahr";
-        last SWITCH;
-      };
-      $monat eq "8" && do {
-        $form->{fromdate} = "1.8.$jahr";
-        $form->{todate}   = "31.8.$jahr";
-        last SWITCH;
-      };
-      $monat eq "9" && do {
-        $form->{fromdate} = "1.9.$jahr";
-        $form->{todate}   = "30.9.$jahr";
-        last SWITCH;
-      };
-      $monat eq "10" && do {
-        $form->{fromdate} = "1.10.$jahr";
-        $form->{todate}   = "31.10.$jahr";
-        last SWITCH;
-      };
-      $monat eq "11" && do {
-        $form->{fromdate} = "1.11.$jahr";
-        $form->{todate}   = "30.11.$jahr";
-        last SWITCH;
-      };
-      $monat eq "12" && do {
-        $form->{fromdate} = "1.12.$jahr";
-        $form->{todate}   = "31.12.$jahr";
-        last SWITCH;
-      };
-    }
-    $fromto .=
-      "'" . $form->{fromdate} . "' and transdate <= '" . $form->{todate} . "'";
-  }
+    # get encoding from defaults - use cp1252 if DATEV strict export is used
+    my $enc = ($::instance_conf->get_datev_export_format eq 'cp1252') ? 'cp1252' : 'utf-8';
+    my $csv_file = IO::File->new($self->export_path . '/' . $filename, ">:encoding($enc)") or die "Can't open: $!";
+
+    $csv->print($csv_file, $_) for @{ $datev_csv->header };
+    $csv->print($csv_file, $_) for @{ $datev_csv->lines  };
+    $csv_file->close;
+    $self->{warnings} = $datev_csv->warnings;
 
-  elsif ($zeitraum eq "quartal") {
-    if ($quartal == 1) {
-      $fromto .=
-        "'01.01." . $jahr . "' and transdate <= '31.03." . $jahr . "'";
-    } elsif ($quartal == 2) {
-      $fromto .=
-        "'01.04." . $jahr . "' and transdate <= '30.06." . $jahr . "'";
-    } elsif ($quartal == 3) {
-      $fromto .=
-        "'01.07." . $jahr . "' and transdate <= '30.09." . $jahr . "'";
-    } elsif ($quartal == 4) {
-      $fromto .=
-        "'01.10." . $jahr . "' and transdate <= '31.12." . $jahr . "'";
+    # convert utf-8 to cp1252//translit if set
+    if ($::instance_conf->get_datev_export_format eq 'cp1252-translit') {
+
+      my $filename_translit = "EXTF_DATEV_kivitendo_translit" . $self->from->ymd() . '-' . $self->to->ymd() . ".csv";
+      open my $fh_in,  '<:encoding(UTF-8)',  $self->export_path . '/' . $filename or die "could not open $filename for reading: $!";
+      open my $fh_out, '>', $self->export_path . '/' . $filename_translit         or die "could not open $filename_translit for writing: $!";
+
+      my $converter = SL::Iconv->new("utf-8", "cp1252//translit");
+
+      print $fh_out $converter->convert($_) while <$fh_in>;
+      close $fh_in;
+      close $fh_out;
+
+      unlink $self->export_path . '/' . $filename or warn "Could not unlink $filename: $!";
+      $filename = $filename_translit;
     }
-  }
 
-  elsif ($zeitraum eq "zeit") {
-    $fromto            .= "'" . $transdatefrom . "' and transdate <= '" . $transdateto . "'";
-    my ($yy, $mm, $dd)  = $main::locale->parse_date(\%main::myconfig, $transdatefrom);
-    $jahr               = $yy;
+    return { download_token => $self->download_token, filenames => $filename };
+
+  } elsif ($self->exporttype == DATEV_ET_STAMM) {
+    die 'will never be implemented';
+    # 'Background: Export should only contain non
+    #  DATEV-Charts and DATEV import will only
+    #  import new Charts.'
+  } elsif ($self->exporttype == DATEV_ET_CSV) {
+    $result = $self->csv_export_for_tax_accountant;
+  } else {
+    die 'unrecognized exporttype';
   }
 
-  $main::lxdebug->leave_sub();
+  return $result;
+}
 
-  return ($fromto, $jahr);
+sub obe_export {
+  die 'not yet implemented';
+}
+
+sub fromto {
+  my ($self) = @_;
+
+  return unless $self->from && $self->to;
+
+  return "transdate >= '" . $self->from->to_lxoffice . "' and transdate <= '" . $self->to->to_lxoffice . "'";
 }
 
 sub _sign {
-  my $value = shift;
+  $_[0] <=> 0;
+}
+
+sub locked {
+ my $self = shift;
 
-  return $value < 0 ? -1
-    :    $value > 0 ?  1
-    :                  0;
+ if (@_) {
+   $self->{locked} = $_[0];
+ }
+ return $self->{locked};
 }
 
-sub _get_transactions {
+sub generate_datev_data {
   $main::lxdebug->enter_sub();
 
-  my $fromto   =  shift;
+  my ($self, %params)   = @_;
+  my $fromto            = $params{from_to} // '';
+  my $progress_callback = $params{progress_callback} || sub {};
 
-  my $myconfig =  \%main::myconfig;
   my $form     =  $main::form;
 
-  my $dbh      =  $form->get_standard_dbh($myconfig);
+  my $trans_id_filter = '';
+  my $ar_department_id_filter = '';
+  my $ap_department_id_filter = '';
+  my $gl_department_id_filter = '';
+  if ( $form->{department_id} ) {
+    $ar_department_id_filter = " AND ar.department_id = ? ";
+    $ap_department_id_filter = " AND ap.department_id = ? ";
+    $gl_department_id_filter = " AND gl.department_id = ? ";
+  }
 
-  my ($notsplitindex);
-  my @errors   = ();
+  my ($gl_itime_filter, $ar_itime_filter, $ap_itime_filter);
+  if ( $form->{gldatefrom} ) {
+    $gl_itime_filter = " AND gl.itime >= ? ";
+    $ar_itime_filter = " AND ar.itime >= ? ";
+    $ap_itime_filter = " AND ap.itime >= ? ";
+  } else {
+    $gl_itime_filter = "";
+    $ar_itime_filter = "";
+    $ap_itime_filter = "";
+  }
 
-  $form->{net_gross_differences}     = [];
-  $form->{sum_net_gross_differences} = 0;
+  if ( $self->{trans_id} ) {
+    # ignore dates when trans_id is passed so that the entire transaction is
+    # checked, not just either the initial bookings or the subsequent payments
+    # (the transdates will likely differ)
+    $fromto = '';
+    $trans_id_filter = 'ac.trans_id = ' . $self->trans_id;
+  } else {
+    $fromto      =~ s/transdate/ac\.transdate/g;
+  };
 
-  $fromto      =~ s/transdate/ac\.transdate/g;
+  my ($notsplitindex);
 
-  my $taxkeys  = Taxkeys->new();
   my $filter   = '';            # Useful for debugging purposes
 
-  my %all_taxchart_ids = selectall_as_map($form, $dbh, qq|SELECT DISTINCT chart_id, TRUE AS is_set FROM tax|, 'chart_id', 'is_set');
+  my %all_taxchart_ids = selectall_as_map($form, $self->dbh, qq|SELECT DISTINCT chart_id, TRUE AS is_set FROM tax|, 'chart_id', 'is_set');
+
+  my $ar_accno = "c.accno";
+  my $ap_accno = "c.accno";
+  if ( $self->use_pk ) {
+    $ar_accno = "CASE WHEN ac.chart_link = 'AR' THEN ct.customernumber ELSE c.accno END as accno";
+    $ap_accno = "CASE WHEN ac.chart_link = 'AP' THEN ct.vendornumber   ELSE c.accno END as accno";
+  }
 
   my $query    =
-    qq|SELECT ac.acc_trans_id, ac.transdate, ac.trans_id,ar.id, ac.amount, ac.taxkey,
-         ar.invnumber, ar.duedate, ar.amount as umsatz,
-         ct.name,
-         c.accno, c.taxkey_id as charttax, c.datevautomatik, c.id, c.link,
-         ar.invoice
+    qq|SELECT ac.acc_trans_id, ac.transdate, ac.gldate, ac.trans_id,ar.id, ac.amount, ac.taxkey, ac.memo,
+         ar.invnumber, ar.duedate, ar.amount as umsatz, ar.deliverydate, ar.itime::date,
+         ct.name, ct.ustid, ct.customernumber AS vcnumber, ct.id AS customer_id, NULL AS vendor_id,
+         $ar_accno, c.description AS accname, c.taxkey_id as charttax, c.datevautomatik, c.id, ac.chart_link AS link,
+         ar.invoice,
+         t.rate AS taxrate, t.taxdescription,
+         'ar' as table,
+         tc.accno AS tax_accno, tc.description AS tax_accname,
+         ar.department_id,
+         ar.notes,
+         project.projectnumber as projectnumber, project.description as projectdescription,
+         department.description as departmentdescription
        FROM acc_trans ac
        LEFT JOIN ar          ON (ac.trans_id    = ar.id)
        LEFT JOIN customer ct ON (ar.customer_id = ct.id)
        LEFT JOIN chart c     ON (ac.chart_id    = c.id)
+       LEFT JOIN tax t       ON (ac.tax_id      = t.id)
+       LEFT JOIN chart tc    ON (t.chart_id     = tc.id)
+       LEFT JOIN department  ON (department.id  = ar.department_id)
+       LEFT JOIN project     ON (project.id     = ar.globalproject_id)
        WHERE (ar.id IS NOT NULL)
          AND $fromto
+         $trans_id_filter
+         $ar_itime_filter
+         $ar_department_id_filter
          $filter
 
        UNION ALL
 
-       SELECT ac.acc_trans_id, ac.transdate, ac.trans_id,ap.id, ac.amount, ac.taxkey,
-         ap.invnumber, ap.duedate, ap.amount as umsatz,
-         ct.name,
-         c.accno, c.taxkey_id as charttax, c.datevautomatik, c.id, c.link,
-         ap.invoice
+       SELECT ac.acc_trans_id, ac.transdate, ac.gldate, ac.trans_id,ap.id, ac.amount, ac.taxkey, ac.memo,
+         ap.invnumber, ap.duedate, ap.amount as umsatz, ap.deliverydate, ap.itime::date,
+         ct.name, ct.ustid, ct.vendornumber AS vcnumber, NULL AS customer_id, ct.id AS vendor_id,
+         $ap_accno, c.description AS accname, c.taxkey_id as charttax, c.datevautomatik, c.id, ac.chart_link AS link,
+         ap.invoice,
+         t.rate AS taxrate, t.taxdescription,
+         'ap' as table,
+         tc.accno AS tax_accno, tc.description AS tax_accname,
+         ap.department_id,
+         ap.notes,
+         project.projectnumber as projectnumber, project.description as projectdescription,
+         department.description as departmentdescription
        FROM acc_trans ac
        LEFT JOIN ap        ON (ac.trans_id  = ap.id)
        LEFT JOIN vendor ct ON (ap.vendor_id = ct.id)
        LEFT JOIN chart c   ON (ac.chart_id  = c.id)
+       LEFT JOIN tax t     ON (ac.tax_id    = t.id)
+       LEFT JOIN chart tc    ON (t.chart_id     = tc.id)
+       LEFT JOIN department  ON (department.id  = ap.department_id)
+       LEFT JOIN project     ON (project.id     = ap.globalproject_id)
        WHERE (ap.id IS NOT NULL)
          AND $fromto
+         $trans_id_filter
+         $ap_itime_filter
+         $ap_department_id_filter
          $filter
 
        UNION ALL
 
-       SELECT ac.acc_trans_id, ac.transdate, ac.trans_id,gl.id, ac.amount, ac.taxkey,
-         gl.reference AS invnumber, gl.transdate AS duedate, ac.amount as umsatz,
-         gl.description AS name,
-         c.accno, c.taxkey_id as charttax, c.datevautomatik, c.id, c.link,
-         FALSE AS invoice
+       SELECT ac.acc_trans_id, ac.transdate, ac.gldate, ac.trans_id,gl.id, ac.amount, ac.taxkey, ac.memo,
+         gl.reference AS invnumber, NULL AS duedate, ac.amount as umsatz, NULL as deliverydate, gl.itime::date,
+         gl.description AS name, NULL as ustid, '' AS vcname, NULL AS customer_id, NULL AS vendor_id,
+         c.accno, c.description AS accname, c.taxkey_id as charttax, c.datevautomatik, c.id, ac.chart_link AS link,
+         FALSE AS invoice,
+         t.rate AS taxrate, t.taxdescription,
+         'gl' as table,
+         tc.accno AS tax_accno, tc.description AS tax_accname,
+         gl.department_id,
+         gl.notes,
+         '' as projectnumber, '' as projectdescription,
+         department.description as departmentdescription
        FROM acc_trans ac
        LEFT JOIN gl      ON (ac.trans_id  = gl.id)
        LEFT JOIN chart c ON (ac.chart_id  = c.id)
+       LEFT JOIN tax t   ON (ac.tax_id    = t.id)
+       LEFT JOIN chart tc    ON (t.chart_id     = tc.id)
+       LEFT JOIN department  ON (department.id  = gl.department_id)
        WHERE (gl.id IS NOT NULL)
          AND $fromto
+         $trans_id_filter
+         $gl_itime_filter
+         $gl_department_id_filter
          $filter
 
        ORDER BY trans_id, acc_trans_id|;
 
-  my $sth = prepare_execute_query($form, $dbh, $query);
-  $form->{DATEV} = [];
+  my @query_args;
+  if ( $form->{gldatefrom} or $form->{department_id} ) {
+
+    for ( 1 .. 3 ) {
+      if ( $form->{gldatefrom} ) {
+        my $glfromdate = $::locale->parse_date_to_object($form->{gldatefrom});
+        die "illegal data" unless ref($glfromdate) eq 'DateTime';
+        push(@query_args, $glfromdate);
+      }
+      if ( $form->{department_id} ) {
+        push(@query_args, $form->{department_id});
+      }
+    }
+  }
+
+  my $sth = prepare_execute_query($form, $self->dbh, $query, @query_args);
+  $self->{DATEV} = [];
 
   my $counter = 0;
-  while (my $ref = $sth->fetchrow_hashref("NAME_lc")) {
+  my $continue = 1; #
+  my $name;
+  while ( $continue && (my $ref = $sth->fetchrow_hashref("NAME_lc")) ) {
+    last unless $ref;  # for single transactions
     $counter++;
     if (($counter % 500) == 0) {
-      print("$counter ");
+      $progress_callback->($counter);
     }
 
     my $trans    = [ $ref ];
 
     my $count    = $ref->{amount};
     my $firstrun = 1;
+
+    # if the amount of a booking in a group is smaller than 0.02, any tax
+    # amounts will likely be smaller than 1 cent, so go into subcent mode
     my $subcent  = abs($count) < 0.02;
 
+    # records from acc_trans are ordered by trans_id and acc_trans_id
+    # first check for unbalanced ledger inside one trans_id
+    # there may be several groups inside a trans_id, e.g. the original booking and the payment
+    # each group individually should be exactly balanced and each group
+    # individually needs its own datev lines
+
+    # keep fetching new acc_trans lines until the end of a balanced group is reached
     while (abs($count) > 0.01 || $firstrun || ($subcent && abs($count) > 0.005)) {
       my $ref2 = $sth->fetchrow_hashref("NAME_lc");
-      last unless ($ref2);
+      unless ( $ref2 ) {
+        $continue = 0;
+        last;
+      };
+
+      # check if trans_id of current acc_trans line is still the same as the
+      # trans_id of the first line in group, i.e. we haven't finished a 0-group
+      # before moving on to the next trans_id, error will likely be in the old
+      # trans_id.
 
       if ($ref2->{trans_id} != $trans->[0]->{trans_id}) {
-        $form->error("Unbalanced ledger! old trans_id " . $trans->[0]->{trans_id} . " new trans_id " . $ref2->{trans_id} . " count $count");
-        ::end_of_request();
+        require SL::DB::Manager::AccTransaction;
+        if ( $trans->[0]->{trans_id} ) {
+          my $acc_trans_obj  = SL::DB::Manager::AccTransaction->get_first(where => [ trans_id => $trans->[0]->{trans_id} ]);
+          $self->add_error(t8("Export error in transaction #1: Unbalanced ledger before next transaction (#2)",
+                              $acc_trans_obj->transaction_name, $ref2->{trans_id})
+          );
+        };
+        return;
       }
 
       push @{ $trans }, $ref2;
@@ -429,20 +687,34 @@ sub _get_transactions {
       if (   !$ref->{invoice}   # we have a non-invoice booking (=gl)
           &&  $ref->{is_tax}    # that has "is_tax" set
           && !($prev_ref->{is_tax})  # previous line wasn't is_tax
-          &&  (_sign($ref->{amount}) == _sign($prev_ref->{amount})))  # and sign same as previous sign
-          {
+          &&  (_sign($ref->{amount}) == _sign($prev_ref->{amount}))) {  # and sign same as previous sign
         $trans->[$i - 1]->{tax_amount} = $ref->{amount};
       }
     }
 
-    my %taxid_taxkeys = ();
     my $absumsatz     = 0;
     if (scalar(@{$trans}) <= 2) {
-      push @{ $form->{DATEV} }, $trans;
+      push @{ $self->{DATEV} }, $trans;
       next;
     }
 
+    # determine at which array position the reference value (called absumsatz) is
+    # and which amount it has
+
     for my $j (0 .. (scalar(@{$trans}) - 1)) {
+
+      # Three cases:
+      # 1: gl transaction (Dialogbuchung), invoice is false, no double split booking allowed
+
+      # 2: sales or vendor invoice (Verkaufs- und Einkaufsrechnung): invoice is
+      # true, instead of absumsatz use link AR/AP (there should only be one
+      # entry)
+
+      # 3. AR/AP transaction (Kreditoren- und Debitorenbuchung): invoice is false,
+      # instead of absumsatz use link AR/AP (there should only be one, so jump
+      # out of search as soon as you find it )
+
+      # case 1 and 2
       # for gl-bookings no split is allowed and there is no AR/AP account, so we always use the maximum value as a reference
       # for ap/ar bookings we can always search for AR/AP in link and use that
       if ( ( not $trans->[$j]->{'invoice'} and abs($trans->[$j]->{'amount'}) > abs($absumsatz) )
@@ -450,12 +722,28 @@ sub _get_transactions {
         $absumsatz     = $trans->[$j]->{'amount'};
         $notsplitindex = $j;
       }
+
+      # case 3
+      # Problem: we can't distinguish between AR and AP and normal invoices via boolean "invoice"
+      # for AR and AP transaction exit the loop as soon as an AR or AP account is found
+      # there must be only one AR or AP chart in the booking
+      # since it is possible to do this kind of things with GL too, make sure those don't get aborted in case someone
+      # manually pays an invoice in GL.
+      if ($trans->[$j]->{table} ne 'gl' and ($trans->[$j]->{'link'} eq 'AR' or $trans->[$j]->{'link'} eq 'AP')) {
+        $notsplitindex = $j;   # position in booking with highest amount
+        $absumsatz     = $trans->[$j]->{'amount'};
+        last;
+      };
     }
 
     my $ml             = ($trans->[0]->{'umsatz'} > 0) ? 1 : -1;
     my $rounding_error = 0;
     my @taxed;
 
+    # go through each line and determine if it is a tax booking or not
+    # skip all tax lines and notsplitindex line
+    # push all other accounts (e.g. income or expense) with corresponding taxkey
+
     for my $j (0 .. (scalar(@{$trans}) - 1)) {
       if (   ($j != $notsplitindex)
           && !$trans->[$j]->{is_tax}
@@ -472,15 +760,14 @@ sub _get_transactions {
         $new_trans{'umsatz'}      = abs($trans->[$j]->{'amount'}) * $ml;
         $trans->[$j]->{'umsatz'}  = abs($trans->[$j]->{'amount'}) * $ml;
 
-        push @{ $form->{DATEV} }, [ \%new_trans, $trans->[$j] ];
+        push @{ $self->{DATEV} }, [ \%new_trans, $trans->[$j] ];
 
       } elsif (($j != $notsplitindex) && !$trans->[$j]->{is_tax}) {
-        my %tax_info = $taxkeys->get_full_tax_info('transdate' => $trans->[$j]->{transdate});
 
         my %new_trans = ();
         map { $new_trans{$_} = $trans->[$notsplitindex]->{$_}; } keys %{ $trans->[$notsplitindex] };
 
-        my $tax_rate              = $tax_info{taxkeys}->{ $trans->[$j]->{'taxkey'} }->{taxrate};
+        my $tax_rate              = $trans->[$j]->{'taxrate'};
         $new_trans{'net_amount'}  = $trans->[$j]->{'amount'} * -1;
         $new_trans{'tax_rate'}    = 1 + $tax_rate;
 
@@ -501,8 +788,8 @@ sub _get_transactions {
           $absumsatz               -= $rounded;
         }
 
-        push @{ $form->{DATEV} }, [ \%new_trans, $trans->[$j] ];
-        push @taxed, $form->{DATEV}->[-1];
+        push @{ $self->{DATEV} }, [ \%new_trans, $trans->[$j] ];
+        push @taxed, $self->{DATEV}->[-1];
       }
     }
 
@@ -546,92 +833,58 @@ sub _get_transactions {
 
     $absumsatz = $form->round_amount($absumsatz, 2);
     if (abs($absumsatz) >= (0.01 * (1 + scalar @taxed))) {
-      push @errors, "Datev-Export fehlgeschlagen! Bei Transaktion $trans->[0]->{trans_id} ($absumsatz)\n";
-
+      require SL::DB::Manager::AccTransaction;
+      my $acc_trans_obj  = SL::DB::Manager::AccTransaction->get_first(where => [ trans_id => $trans->[0]->{trans_id} ]);
+      $self->add_error(t8("Export error in transaction #1: Rounding error too large #2",
+                          $acc_trans_obj->transaction_name, $absumsatz)
+      );
     } elsif (abs($absumsatz) >= 0.01) {
-      push @{ $form->{net_gross_differences} }, $absumsatz;
-      $form->{sum_net_gross_differences} += $absumsatz;
+      $self->add_net_gross_differences($absumsatz);
     }
   }
 
   $sth->finish();
 
-  $form->error(join("
\n", @errors)) if (@errors);
-
-  $main::lxdebug->leave_sub();
+  $::lxdebug->leave_sub;
 }
 
 sub make_kne_data_header {
   $main::lxdebug->enter_sub();
 
-  my ($myconfig, $form, $fromto, $start_jahr) = @_;
+  my ($self, $form) = @_;
   my ($primanota);
 
-  my $jahr = $start_jahr;
-  if (!$jahr) {
-    my @a = localtime;
-    $jahr = $a[5];
-  }
+  my $stamm = $self->get_datev_stamm;
+
+  my $jahr = $self->from ? $self->from->year : DateTime->today->year;
 
   #Header
   my $header  = "\x1D\x181";
-  $header    .= _fill($form->{datentraegernr}, 3, ' ', 'left');
-  $header    .= ($fromto) ? "11" : "13"; # Anwendungsnummer
-  $header    .= _fill($form->{dfvkz}, 2, '0');
-  $header    .= _fill($form->{beraternr}, 7, '0');
-  $header    .= _fill($form->{mandantennr}, 5, '0');
-  $header    .= _fill($form->{abrechnungsnr} . $jahr, 6, '0');
-
-  $fromto         =~ s/transdate|>=|and|\'|<=//g;
-  my ($from, $to) =  split /   /, $fromto;
-  $from           =~ s/ //g;
-  $to             =~ s/ //g;
-
-  if ($from ne "") {
-    my ($fday, $fmonth, $fyear) = split(/\./, $from);
-    if (length($fmonth) < 2) {
-      $fmonth = "0" . $fmonth;
-    }
-    if (length($fday) < 2) {
-      $fday = "0" . $fday;
-    }
-    $from = $fday . $fmonth . substr($fyear, -2, 2);
-  } else {
-    $from = "";
-  }
+  $header    .= _fill($stamm->{datentraegernr}, 3, ' ', 'left');
+  $header    .= ($self->fromto) ? "11" : "13"; # Anwendungsnummer
+  $header    .= _fill($stamm->{dfvkz}, 2, '0');
+  $header    .= _fill($stamm->{beraternr}, 7, '0');
+  $header    .= _fill($stamm->{mandantennr}, 5, '0');
+  $header    .= _fill(($stamm->{abrechnungsnr} // '') . $jahr, 6, '0');
 
-  $header .= $from;
+  $header .= $self->from ? $self->from->strftime('%d%m%y') : '';
+  $header .= $self->to   ? $self->to->strftime('%d%m%y')   : '';
 
-  if ($to ne "") {
-    my ($tday, $tmonth, $tyear) = split(/\./, $to);
-    if (length($tmonth) < 2) {
-      $tmonth = "0" . $tmonth;
-    }
-    if (length($tday) < 2) {
-      $tday = "0" . $tday;
-    }
-    $to = $tday . $tmonth . substr($tyear, -2, 2);
-  } else {
-    $to = "";
-  }
-  $header .= $to;
-
-  if ($fromto ne "") {
+  if ($self->fromto) {
     $primanota = "001";
     $header .= $primanota;
   }
 
-  $header .= _fill($form->{passwort}, 4, '0');
+  $header .= _fill($stamm->{passwort}, 4, '0');
   $header .= " " x 16;       # Anwendungsinfo
   $header .= " " x 16;       # Inputinfo
   $header .= "\x79";
 
   #Versionssatz
-  my $versionssatz  = $form->{exporttype} == 0 ? "\xB5" . "1," : "\xB6" . "1,";
+  my $versionssatz  = $self->exporttype == DATEV_ET_BUCHUNGEN ? "\xB5" . "1," : "\xB6" . "1,";
 
-  my $dbh           = $form->get_standard_dbh($myconfig);
   my $query         = qq|SELECT accno FROM chart LIMIT 1|;
-  my $ref           = selectfirst_hashref_query($form, $dbh, $query);
+  my $ref           = selectfirst_hashref_query($form, $self->dbh, $query);
 
   $versionssatz    .= length $ref->{accno};
   $versionssatz    .= ",";
@@ -652,9 +905,6 @@ sub datetofour {
 
   my ($day, $month, $year) = split(/\./, $date);
 
-  if ($day =~ /^0/) {
-    $day = substr($day, 1, 1);
-  }
   if (length($month) < 2) {
     $month = "0" . $month;
   }
@@ -684,12 +934,12 @@ sub trim_leading_zeroes {
 sub make_ed_versionset {
   $main::lxdebug->enter_sub();
 
-  my ($header, $filename, $blockcount, $fromto) = @_;
+  my ($self, $header, $filename, $blockcount) = @_;
 
   my $versionset  = "V" . substr($filename, 2, 5);
   $versionset    .= substr($header, 6, 22);
 
-  if ($fromto ne "") {
+  if ($self->fromto) {
     $versionset .= "0000" . substr($header, 28, 19);
   } else {
     my $datum = " " x 16;
@@ -710,12 +960,14 @@ sub make_ed_versionset {
 sub make_ev_header {
   $main::lxdebug->enter_sub();
 
-  my ($form, $fileno) = @_;
+  my ($self, $form, $fileno) = @_;
 
-  my $ev_header  = _fill($form->{datentraegernr}, 3, ' ', 'left');
+  my $stamm = $self->get_datev_stamm;
+
+  my $ev_header  = _fill($stamm->{datentraegernr}, 3, ' ', 'left');
   $ev_header    .= "   ";
-  $ev_header    .= _fill($form->{beraternr}, 7, ' ', 'left');
-  $ev_header    .= _fill($form->{beratername}, 9, ' ', 'left');
+  $ev_header    .= _fill($stamm->{beraternr}, 7, ' ', 'left');
+  $ev_header    .= _fill($stamm->{beratername}, 9, ' ', 'left');
   $ev_header    .= " ";
   $ev_header    .= (_fill($fileno, 5, '0')) x 2;
   $ev_header    .= " " x 95;
@@ -725,195 +977,241 @@ sub make_ev_header {
   return $ev_header;
 }
 
+sub generate_datev_lines {
+  my ($self) = @_;
+
+  my @datev_lines = ();
+
+  foreach my $transaction ( @{ $self->{DATEV} } ) {
+
+    # each $transaction entry contains data from several acc_trans entries
+    # belonging to the same trans_id
+
+    my %datev_data = (); # data for one transaction
+    my $trans_lines = scalar(@{$transaction});
+
+    my $umsatz         = 0;
+    my $gegenkonto     = "";
+    my $konto          = "";
+    my $belegfeld1     = "";
+    my $datum          = "";
+    my $waehrung       = "";
+    my $buchungstext   = "";
+    my $belegfeld2     = "";
+    my $datevautomatik = 0;
+    my $taxkey         = 0;
+    my $charttax       = 0;
+    my $ustid          ="";
+    my ($haben, $soll);
+    for (my $i = 0; $i < $trans_lines; $i++) {
+      if ($trans_lines == 2) {
+        if (abs($transaction->[$i]->{'amount'}) > abs($umsatz)) {
+          $umsatz = $transaction->[$i]->{'amount'};
+        }
+      } else {
+        if (abs($transaction->[$i]->{'umsatz'}) > abs($umsatz)) {
+          $umsatz = $transaction->[$i]->{'umsatz'};
+        }
+      }
+      if ($transaction->[$i]->{'datevautomatik'}) {
+        $datevautomatik = 1;
+      }
+      if ($transaction->[$i]->{'taxkey'}) {
+        $taxkey = $transaction->[$i]->{'taxkey'};
+      }
+      if ($transaction->[$i]->{'charttax'}) {
+        $charttax = $transaction->[$i]->{'charttax'};
+      }
+      if ($transaction->[$i]->{'amount'} > 0) {
+        $haben = $i;
+      } else {
+        $soll = $i;
+      }
+    }
+
+    if ($trans_lines >= 2) {
+
+      # Personenkontenerweiterung: accno has already been replaced if use_pk was set
+      $datev_data{'gegenkonto'} = $transaction->[$haben]->{'accno'};
+      $datev_data{'konto'}      = $transaction->[$soll]->{'accno'};
+      if ($transaction->[$haben]->{'invnumber'} ne "") {
+        $datev_data{belegfeld1} = $transaction->[$haben]->{'invnumber'};
+      }
+      $datev_data{datum} = $transaction->[$haben]->{'transdate'};
+      $datev_data{waehrung} = 'EUR';
+      $datev_data{kost1} = $transaction->[$haben]->{'departmentdescription'};
+      $datev_data{kost2} = $transaction->[$haben]->{'projectdescription'};
+
+      if ($transaction->[$haben]->{'name'} ne "") {
+        $datev_data{buchungstext} = $transaction->[$haben]->{'name'};
+      }
+      if (($transaction->[$haben]->{'ustid'} // '') ne "") {
+        $datev_data{ustid} = SL::VATIDNr->normalize($transaction->[$haben]->{'ustid'});
+      }
+      if (($transaction->[$haben]->{'duedate'} // '') ne "") {
+        $datev_data{belegfeld2} = $transaction->[$haben]->{'duedate'};
+      }
+    }
+    $datev_data{umsatz} = abs($umsatz); # sales invoices without tax have a different sign???
+
+    # Dies ist die einzige Stelle die datevautomatik auswertet. Was soll gesagt werden?
+    # Im Prinzip hat jeder acc_trans Eintrag einen Steuerschlüssel, auÃer, bei gewissen Fällen
+    # wie: Kreditorenbuchung mit negativen Vorzeichen, SEPA-Export oder Rechnungen die per
+    # Skript angelegt werden.
+    # Also falls ein Steuerschlüssel da ist und NICHT datevautomatik diesen Block hinzufügen.
+    # Oder aber datevautomatik ist WAHR, aber der Steuerschlüssel in der acc_trans weicht
+    # von dem in der Chart ab: Also wahrscheinlich Programmfehler (NULL übergeben, statt
+    # DATEV-Steuerschlüssel) oder der Steuerschlüssel des Kontos weicht WIRKLICH von dem Eintrag in der
+    # acc_trans ab. Gibt es für diesen Fall eine plausiblen Grund?
+    #
+
+    # only set buchungsschluessel if the following conditions are met:
+    if (   ( $datevautomatik || $taxkey)
+        && (!$datevautomatik || ($datevautomatik && ($charttax ne $taxkey)))) {
+      # $datev_data{buchungsschluessel} = !$datevautomatik ? $taxkey : "4";
+      $datev_data{buchungsschluessel} = $taxkey;
+    }
+    # set lock for each transaction
+    $datev_data{locked} = $self->locked;
+
+    push(@datev_lines, \%datev_data) if $datev_data{umsatz};
+  }
+
+  # example of modifying export data:
+  # foreach my $datev_line ( @datev_lines ) {
+  #   if ( $datev_line{"konto"} eq '1234' ) {
+  #     $datev_line{"konto"} = '9999';
+  #   }
+  # }
+  #
+
+  return \@datev_lines;
+}
+
+
 sub kne_buchungsexport {
   $main::lxdebug->enter_sub();
 
-  my ($myconfig, $form) = @_;
+  my ($self) = @_;
+
+  my $form = $::form;
 
   my @filenames;
 
-  my $export_path = _get_export_path() . "/";
-  my $filename    = "ED00000";
+  my $filename    = "ED00001";
   my $evfile      = "EV01";
   my @ed_versionset;
-  my $fileno = 0;
-
-  $form->header;
-  print qq|
-  
-