Merge branch 'b-3.6.1' of ../kivitendo-erp_20220811

[kivitendo-erp.git] / SL / Helper / Csv.pm

diff --git a/SL/Helper/Csv.pm b/SL/Helper/Csv.pm
index 50a95fa..0d471eb 100644 (file)
--- a/SL/Helper/Csv.pm
+++ b/SL/Helper/Csv.pm
@@ -114,6 +114,7 @@ sub _check_multiplexed {
       # Each profile needs a class and a row_ident
       my $info_ok = all { defined $_->{class} && defined $_->{row_ident} } @profile;
       $self->_push_error([
+        undef,
         0,
         "missing class or row_ident in one of the profiles for multiplexed data",
         0,
@@ -125,6 +126,7 @@ sub _check_multiplexed {
         my @header = @{ $self->header };
         my $t_ok = scalar @profile == scalar @header;
         $self->_push_error([
+          undef,
           0,
           "number of headers and number of profiles must be the same for multiplexed data",
           0,
@@ -133,6 +135,7 @@ sub _check_multiplexed {
 
         $t_ok = all { scalar @$_ > 0} @header;
         $self->_push_error([
+          undef,
           0,
           "no empty headers are allowed for multiplexed data",
           0,
@@ -158,10 +161,11 @@ sub _check_header {
     foreach my $p_num (0..$n_header - 1) {
       my $h = $self->_csv->getline($self->_io);
 
+      my ($code, $string, $position, $record, $field) = $self->_csv->error_diag;
+
       $self->_push_error([
         $self->_csv->error_input,
-        $self->_csv->error_diag,
-        0,
+        $code, $string, $position, $record // 0,
       ]) unless $h;
 
       if ($self->is_multiplexed) {
@@ -202,12 +206,13 @@ sub _check_header {
       my $h_aref = ($self->is_multiplexed)? $header : [ $header ];
       my $p_num  = 0;
       foreach my $h (@{ $h_aref }) {
-        my @names = (
-          keys %{ $self->profile->[$p_num]->{profile} || {} },
+        my %names = (
+          (map { $_ => $_                                     } keys %{ $self->profile->[$p_num]->{profile} || {} }),
+          (map { $_ => $self->profile->[$p_num]{mapping}{$_}  } keys %{ $self->profile->[$p_num]->{mapping} || {} }),
         );
-        for my $name (@names) {
+        for my $name (keys %names) {
           for my $i (0..$#$h) {
-            $h->[$i] = $name if lc $h->[$i] eq lc $name;
+            $h->[$i] = $names{$name} if lc $h->[$i] eq lc $name;
           }
         }
         $p_num++;
@@ -229,7 +234,8 @@ sub _check_multiplex_datatype_position {
     $self->_multiplex_datatype_position($first_pos);
     return 1;
   } else {
-    $self->_push_error([0,
+    $self->_push_error([undef,
+                        0,
                         "datatype field must be at the same position for all datatypes for multiplexed data",
                         0,
                         0]);
@@ -237,6 +243,10 @@ sub _check_multiplex_datatype_position {
   }
 }
 
+sub _is_empty_row {
+  return !!all { !$_ } @{$_[0]};
+}
+
 sub _parse_data {
   my ($self, %params) = @_;
   my (@data, @errors);
@@ -244,9 +254,11 @@ sub _parse_data {
   while (1) {
     my $row = $self->_csv->getline($self->_io);
     if ($row) {
+      next if _is_empty_row($row);
       my $header = $self->_header_by_row($row);
       if (!$header) {
         push @errors, [
+          undef,
           0,
           "Cannot get header for row. Maybe row name and datatype field not matching.",
           0,
@@ -258,19 +270,15 @@ sub _parse_data {
       push @data, \%hr;
     } else {
       last if $self->_csv->eof;
+
       # Text::CSV_XS 0.89 added record number to error_diag
-      if (qv(Text::CSV_XS->VERSION) >= qv('0.89')) {
-        push @errors, [
-          $self->_csv->error_input,
-          $self->_csv->error_diag,
-        ];
-      } else {
-        push @errors, [
-          $self->_csv->error_input,
-          $self->_csv->error_diag,
-          $self->_io->input_line_number,
-        ];
-      }
+      my ($code, $string, $position, $record, $field) = $self->_csv->error_diag;
+
+      push @errors, [
+        $self->_csv->error_input,
+        $code, $string, $position,
+        $record // $self->_io->input_line_number,
+      ];
     }
     last if $self->_csv->eof;
   }
@@ -340,6 +348,9 @@ sub _push_error {
   $self->_errors(\@new_errors);
 }
 
+sub specs {
+  $_[0]->dispatcher->_specs
+}
 
 1;
 
@@ -496,7 +507,7 @@ See section L</PROFILE> for information on this topic.
 
 =item C<ignore_unknown_columns>
 
-If set, the import will ignore unkown header columns. Useful for lazy imports,
+If set, the import will ignore unknown header columns. Useful for lazy imports,
 but deactivated by default.
 
 =item C<case_insensitive_header>
@@ -527,22 +538,20 @@ The basic structure is:
 
   PROFILE       := [ CLASS_PROFILE, CLASS_PROFILE* ]
   CLASS_PROFILE := {
-                      profile   => { ACCESSORS },
+                      profile   => { ACCESSORS+ },
                       class     => $classname,
                       row_ident => $row_ident,
+                      mapping   => { MAPPINGS* },
                    }
-  ACCESSORS     := $field => $accessor, ACCESSORS*
+  ACCESSORS     := $field => $accessor
+  MAPPINGS      := $alias => $field
 
-The C<profile> is a HASHREF which may be used to map header fields to custom
+The C<ACCESSORS> may be used to map header fields to custom
 accessors. Example:
 
-  [
-    {
-      profile => {
-        listprice => 'listprice_as_number',
-      }
-    }
-  ]
+  profile => {
+    listprice => 'listprice_as_number',
+  }
 
 In this case C<listprice_as_number> will be used to store the values from the
 C<listprice> column.
@@ -575,7 +584,35 @@ dispatching into it.
 C<row_ident> is used to determine the correct profile in multiplexed data and
 must be given there. It's not used in non-multiplexed data.
 
-Example:
+If C<mappings> is present, it must contain a hashref that maps strings to known
+fields. This can be used to add custom profiles for known sources, that don't
+comply with the expected header identities.
+
+Without strict profiles, mappings can also directly map header fields that
+should end up in the same accessor.
+
+With case insensitive headings, mappings will also modify the headers, to fit
+the expected profile.
+
+Mappings can be identical to known fields and will be prefered during lookup,
+but will not replace the field, meaning that:
+
+  profile => {
+    name        => 'name',
+    description => 'description',
+  }
+  mapping => {
+    name        => 'description',
+    shortname   => 'name',
+  }
+
+will work as expected, and shortname will not end up in description. This also
+works with the case insensitive option. Note however that the case insensitive
+option will not enable true unicode collating.
+
+
+Here's a full example:
+
   [
     {
       class     => 'SL::DB::Order',
@@ -584,7 +621,8 @@ Example:
     {
       class     => 'SL::DB::OrderItem',
       row_ident => 'I',
-      profile   => { sellprice => 'sellprice_as_number' }
+      profile   => { sellprice => 'sellprice_as_number' },
+      mapping   => { 'Verkaufspreis' => 'sellprice' }
     },
   ]
 
@@ -601,6 +639,9 @@ Each entry is an object with the following attributes:
 
 Note that the last entry can be off, but will give an estimate.
 
+Error handling is also known to break on new Perl versions and need to be
+adjusted from time to time due to changes in Text::CSV_XS.
+
 =head1 CAVEATS
 
 =over 4