From 324726acd30b8992854a2d59fec2a50265613eef Mon Sep 17 00:00:00 2001 From: =?utf8?q?Jan=20B=C3=BCren?= Date: Thu, 29 Nov 2018 14:45:33 +0100 Subject: [PATCH] Fixt #348 DatevExport kommt mit bestimmten Zeichen im Buchungstext nicht klar In der Mandantenkonfiguration befindet sich jetzt eine Einstellung, welche die Kodierung des DATEV-Exports steuert. DATEV erwartet CP1252. kivitendo kann diese Kodierung so vom kivitendo Nutzer einfordern, alternativ nicht vorhandenen Zeichen versuchen zu ersetzen oder die DATEV-Erwartung ignorieren und UTF-8 liefern. Voreingestellt ist CP1252 mit Ersetzungen --- SL/DATEV.pm | 23 ++++++++++++++++++- SL/DATEV/CSV.pm | 11 ++++++--- SL/DB/MetaSetup/Default.pm | 1 + locale/de/all | 7 +++++- sql/Pg-upgrade2/datev_export_format.sql | 8 +++++++ .../_datev_check_configuration.html | 5 ++++ templates/webpages/client_config/form.html | 2 +- 7 files changed, 51 insertions(+), 6 deletions(-) create mode 100644 sql/Pg-upgrade2/datev_export_format.sql diff --git a/SL/DATEV.pm b/SL/DATEV.pm index ebff4fc54..51fe4bde5 100644 --- a/SL/DATEV.pm +++ b/SL/DATEV.pm @@ -35,6 +35,7 @@ use SL::DATEV::KNEFile; use SL::DATEV::CSV; use SL::DB; use SL::HTML::Util (); +use SL::Iconv; use SL::Locale::String qw(t8); use Data::Dumper; @@ -393,12 +394,32 @@ sub csv_export { eol => "\r\n", }) or die "Cannot use CSV: ".Text::CSV_XS->error_diag(); - my $csv_file = IO::File->new($self->export_path . '/' . $filename, '>:encoding(cp1252)') or die "Can't open: $!"; + # get encoding from defaults - use cp1252 if DATEV strict export is used + my $enc = ($::instance_conf->get_datev_export_format eq 'cp1252') ? 'cp1252' : 'utf-8'; + my $csv_file = IO::File->new($self->export_path . '/' . $filename, ">:encoding($enc)") or die "Can't open: $!"; + $csv->print($csv_file, $_) for @{ $datev_csv->header }; $csv->print($csv_file, $_) for @{ $datev_csv->lines }; $csv_file->close; $self->{warnings} = $datev_csv->warnings; + # convert utf-8 to cp1252//translit if set + if ($::instance_conf->get_datev_export_format eq 'cp1252-translit') { + + my $filename_translit = "EXTF_DATEV_kivitendo_translit" . $self->from->ymd() . '-' . $self->to->ymd() . ".csv"; + open my $fh_in, '<:encoding(UTF-8)', $self->export_path . '/' . $filename or die "could not open $filename for reading: $!"; + open my $fh_out, '>', $self->export_path . '/' . $filename_translit or die "could not open $filename_translit for writing: $!"; + + my $converter = SL::Iconv->new("utf-8", "cp1252//translit"); + + print $fh_out $converter->convert($_) while <$fh_in>; + close $fh_in; + close $fh_out; + + unlink $self->export_path . '/' . $filename or warn "Could not unlink $filename: $!"; + $filename = $filename_translit; + } + return { download_token => $self->download_token, filenames => $filename }; } elsif ($self->exporttype == DATEV_ET_STAMM) { diff --git a/SL/DATEV/CSV.pm b/SL/DATEV/CSV.pm index 1f31f169b..e24755b89 100644 --- a/SL/DATEV/CSV.pm +++ b/SL/DATEV/CSV.pm @@ -103,7 +103,10 @@ my @kivitendo_to_datev = ( max_length => 12, type => 'Text', default => '', - input_check => sub { my ($text) = @_; check_encoding($text); }, + input_check => sub { return 1 unless $::instance_conf->get_datev_export_format eq 'cp1252'; + my ($text) = @_; check_encoding($text); }, + valid_check => sub { return 1 if $::instance_conf->get_datev_export_format eq 'cp1252'; + my ($text) = @_; check_encoding($text); }, formatter => sub { my ($input) = @_; return substr($input, 0, 12) }, }, { @@ -127,8 +130,10 @@ my @kivitendo_to_datev = ( max_length => 60, type => 'Text', default => '', - input_check => sub { my ($text) = @_; return 1 unless $text; check_encoding($text); }, - formatter => sub { my ($input) = @_; return substr($input, 0, 60) }, + input_check => sub { return 1 unless $::instance_conf->get_datev_export_format eq 'cp1252'; + my ($text) = @_; check_encoding($text); }, + valid_check => sub { return 1 if $::instance_conf->get_datev_export_format eq 'cp1252'; + my ($text) = @_; check_encoding($text); }, }, # pos 14 { kivi_datev_name => 'not yet implemented', diff --git a/SL/DB/MetaSetup/Default.pm b/SL/DB/MetaSetup/Default.pm index 1dff040f3..b948198bc 100644 --- a/SL/DB/MetaSetup/Default.pm +++ b/SL/DB/MetaSetup/Default.pm @@ -45,6 +45,7 @@ __PACKAGE__->meta->columns( datev_check_on_gl_transaction => { type => 'boolean', default => 'true' }, datev_check_on_purchase_invoice => { type => 'boolean', default => 'true' }, datev_check_on_sales_invoice => { type => 'boolean', default => 'true' }, + datev_export_format => { type => 'enum', check_in => [ 'cp1252', 'cp1252-translit', 'utf-8' ], db_type => 'datev_export_format_enum', default => 'cp1252-translit' }, disabled_price_sources => { type => 'array' }, doc_delete_printfiles => { type => 'boolean', default => 'false' }, doc_files => { type => 'boolean', default => 'false' }, diff --git a/locale/de/all b/locale/de/all index e99d817a7..0967e474e 100755 --- a/locale/de/all +++ b/locale/de/all @@ -844,8 +844,9 @@ $self->{texts} = { 'DATEV - Export Assistent' => 'DATEV-Exportassistent', 'DATEV Angaben' => 'DATEV-Angaben', 'DATEV Export' => 'DATEV-Export', - 'DATEV check configuration' => 'Einstellungen für DATEV-Prüfung', 'DATEV check returned errors:' => 'Die DATEV Prüfung dieser Buchung ergab Fehler:', + 'DATEV configuration' => 'Einstellungen für DATEV', + 'DATEV expects the encoding to be Western Europe conform (LATIN-1, cp1252). By setting this to "Strict and halt" the DATEV export halts with a error if there is a single character in "Posting Text" which is not LATIN-1 encodeable. By setting this to "Strict but replace" kivitendo will replace the character with a similar one and the export will simply warn about those fields. By setting this to relaxed (UTF-8) the DATEV export encoding will be in kivitendo (UTF-8) encoded and the external import program has to handle this (this may work for DATEV deriviates or future versions of DATEV). Background details: For example turkish characters (Ç) are not valid cp1252 charactes and armenian characters like "Գեղարդ" are probably not replaceable in cp1252' => 'DATEV erwartet westeuropäische Zeichenkodierung (LATIN-1, cp1252). Die Einstellung "Strikt und Abbruch" erlaubt keine nicht kodiebaren Zeichen im DATEV-Export und bricht diesen mit einer Fehlermeldung ab. Die Einstellung "Strikt mit Ersetzungen" versucht ähnliche Zeichen (bspw. c statt ć) zu verwenden und gibt zusätzlich eine Warnung beim DATEV-Expport aus. Die Einstellung "Lax (UTF-8)" ignoriert diese Anforderung und übergibt die Daten im kivitendo-konformen UTF-8 Format. Letzteres kann für zukünftige DATEV-Version oder DATEV-kompatible Alternativen interessant sein. Hintergrund-Info: Beispielsweise sind schon türkische Zeichen (Ç) nicht mehr im westeuropäischen Zeichensätze enthalten und armenische Zeiche wie "Գեղարդ" könnnen sicherlich überhaupt nicht mit Zeichen in cp1252 ersetzt werden.', 'DATEX - Export Assistent' => 'DATEV-Exportassistent', 'DELETED' => 'Gelöscht', 'DFV-Kennzeichen' => 'DFV-Kennzeichen', @@ -882,6 +883,7 @@ $self->{texts} = { 'Date missing!' => 'Datum fehlt!', 'Date the payment is due in full' => 'Das Datum, bis die Rechnung in voller Höhe bezahlt werden muss', 'Date the payment is due with discount' => 'Das Datum, bis die Rechnung unter Abzug von Skonto bezahlt werden kann', + 'Datev export encoding' => 'DATEV-Export Kodierung', 'Datevautomatik' => 'Datev-Automatik', 'Datum von' => 'Datum von', 'Deactivate by default' => 'Deaktiviert als Voreinstellung', @@ -2530,6 +2532,7 @@ $self->{texts} = { 'Reference / Invoice Number' => 'Referenz / Rechnungsnummer', 'Reference day' => 'Referenztag', 'Reference missing!' => 'Referenz fehlt!', + 'Relaxed (UTF-8)' => 'Lax (UTF-8)', 'Release From Stock' => 'Lagerausgang', 'Remaining' => 'Rest', 'Remaining Amount' => 'abzurechnender Betrag', @@ -2991,6 +2994,8 @@ $self->{texts} = { 'Storno (one letter abbreviation)' => 'S', 'Storno Invoice' => 'Stornorechnung', 'Street' => 'Straße', + 'Strict and halt' => 'Strikt und Abbruch', + 'Strict but replace' => 'Strikt mit Ersetzungen', 'Style the picture with the following CSS code' => 'Bildeigenschaft mit folgendem CSS-Style versehen', 'Stylesheet' => 'Stilvorlage', 'Sub function blocks' => 'Unterfunktionsblöcke', diff --git a/sql/Pg-upgrade2/datev_export_format.sql b/sql/Pg-upgrade2/datev_export_format.sql new file mode 100644 index 000000000..c4d79f7fc --- /dev/null +++ b/sql/Pg-upgrade2/datev_export_format.sql @@ -0,0 +1,8 @@ +-- @tag: datev_export_format +-- @description: Setzt die ausgehende Formatierung des DATEV-Exports +-- @depends: release_3_5_1 + +CREATE TYPE datev_export_format_enum AS ENUM ('cp1252', 'cp1252-translit', 'utf-8'); + +ALTER TABLE defaults ADD COLUMN datev_export_format datev_export_format_enum default 'cp1252-translit'; + diff --git a/templates/webpages/client_config/_datev_check_configuration.html b/templates/webpages/client_config/_datev_check_configuration.html index ec1b433d5..dc99c218e 100644 --- a/templates/webpages/client_config/_datev_check_configuration.html +++ b/templates/webpages/client_config/_datev_check_configuration.html @@ -29,5 +29,10 @@ [% L.yes_no_tag('defaults.datev_check_on_gl_transaction', SELF.defaults.datev_check_on_gl_transaction) %] [% LxERP.t8('Perform check when a gl transaction is posted?') %] + + [% LxERP.t8('Datev export encoding') %] + [% L.select_tag('defaults.datev_export_format', [ [ 'cp1252', LxERP.t8('Strict and halt') ],[ 'cp1252-translit', LxERP.t8('Strict but replace') ],[ 'utf-8', LxERP.t8('Relaxed (UTF-8)') ] ], default=SELF.defaults.datev_export_format) %] + [% LxERP.t8('DATEV expects the encoding to be Western Europe conform (LATIN-1, cp1252). By setting this to "Strict and halt" the DATEV export halts with a error if there is a single character in "Posting Text" which is not LATIN-1 encodeable. By setting this to "Strict but replace" kivitendo will replace the character with a similar one and the export will simply warn about those fields. By setting this to relaxed (UTF-8) the DATEV export encoding will be in kivitendo (UTF-8) encoded and the external import program has to handle this (this may work for DATEV deriviates or future versions of DATEV). Background details: For example turkish characters (Ç) are not valid cp1252 charactes and armenian characters like "Գեղարդ" are probably not replaceable in cp1252') %] + diff --git a/templates/webpages/client_config/form.html b/templates/webpages/client_config/form.html index f0d26214b..dcb5576c7 100644 --- a/templates/webpages/client_config/form.html +++ b/templates/webpages/client_config/form.html @@ -84,7 +84,7 @@ $(function() {
  • [% LxERP.t8('Default Accounts') %]
  • [% LxERP.t8('Posting Configuration') %]
    • [% IF FORM.feature_datev %] -
  • [% LxERP.t8('DATEV check configuration') %]
    • +
  • [% LxERP.t8('DATEV configuration') %]
    • [% END %]
  • [% LxERP.t8('Orders / Delivery Orders deleteable') %]
    • [%- IF INSTANCE_CONF.get_doc_storage %] -- 2.20.1