BankTransaction: Verlinkungen auf alten Vorlagencode upgedated.

[mfinanz.git] / SL / Helper / Csv.pm

diff --git a/SL/Helper/Csv.pm b/SL/Helper/Csv.pm
index 85f175889c48d5061f92554a7eced03767db8b00..c58575afe360fccec7d7faa49ee224227e9ee0b0 100644 (file)
--- a/SL/Helper/Csv.pm
+++ b/SL/Helper/Csv.pm
@@ -7,12 +7,13 @@ use version 0.77;
 use Carp;
 use IO::File;
 use Params::Validate qw(:all);
 use Carp;
 use IO::File;
 use Params::Validate qw(:all);

-use List::MoreUtils qw(all pairwise);
+use List::MoreUtils qw(all pairwise firstidx);

 use Text::CSV_XS;
 use Rose::Object::MakeMethods::Generic scalar => [ qw(
   file encoding sep_char quote_char escape_char header profile
   numberformat dateformat ignore_unknown_columns strict_profile is_multiplexed
   _row_header _io _csv _objects _parsed _data _errors all_cvar_configs case_insensitive_header
 use Text::CSV_XS;
 use Rose::Object::MakeMethods::Generic scalar => [ qw(
   file encoding sep_char quote_char escape_char header profile
   numberformat dateformat ignore_unknown_columns strict_profile is_multiplexed
   _row_header _io _csv _objects _parsed _data _errors all_cvar_configs case_insensitive_header

+  _multiplex_datatype_position

 ) ];
 
 use SL::Helper::Csv::Dispatcher;
 ) ];
 
 use SL::Helper::Csv::Dispatcher;


@@ -59,6 +60,7 @@ sub parse {
   $self->_open_file;
   return if ! $self->_check_multiplexed;
   return if ! $self->_check_header;
   $self->_open_file;
   return if ! $self->_check_multiplexed;
   return if ! $self->_check_header;

+  return if ! $self->_check_multiplex_datatype_position;

   return if ! $self->dispatcher->parse_profile;
   return if ! $self->_parse_data;
 
   return if ! $self->dispatcher->parse_profile;
   return if ! $self->_parse_data;
 


@@ -75,7 +77,7 @@ sub get_objects {
   croak 'must parse first' unless $self->_parsed;
 
   $self->_make_objects unless $self->_objects;
   croak 'must parse first' unless $self->_parsed;
 
   $self->_make_objects unless $self->_objects;

-  return wantarray ? @{ $self->_objects } : $self->_objects;
+  return $self->_objects;

 }
 
 sub errors {
 }
 
 sub errors {


@@ -117,7 +119,7 @@ sub _check_multiplexed {
         0,
         0]) unless $info_ok;
 
         0,
         0]) unless $info_ok;
 

-      # If header is given, there need to be a header for each profile
+      # If header is given, there needs to be a header for each profile

       # and no empty headers.
       if ($info_ok && $self->header) {
         my @header = @{ $self->header };
       # and no empty headers.
       if ($info_ok && $self->header) {
         my @header = @{ $self->header };


@@ -162,7 +164,11 @@ sub _check_header {
         0,
       ]) unless $h;
 
         0,
       ]) unless $h;
 

-      push @{ $header }, $h;
+      if ($self->is_multiplexed) {
+        push @{ $header }, $h;
+      } else {
+        $header = $h;
+      }

     }
   }
 
     }
   }
 


@@ -171,31 +177,38 @@ sub _check_header {
   # data with a discouraged but valid byte order mark
   # if not removed, the first header field will not be recognized
   if ($header) {
   # data with a discouraged but valid byte order mark
   # if not removed, the first header field will not be recognized
   if ($header) {

-    my $h = $header->[0];
+    my $h = ($self->is_multiplexed)? $header->[0] : $header;
+

     if ($h && $h->[0] && $self->encoding =~ /utf-?8/i) {
       $h->[0] =~ s/^\x{FEFF}//;
     }
   }
 
   # check, if all header fields are parsed well
     if ($h && $h->[0] && $self->encoding =~ /utf-?8/i) {
       $h->[0] =~ s/^\x{FEFF}//;
     }
   }
 
   # check, if all header fields are parsed well

-  return unless $header && all { $_ } @$header;
+  if ($self->is_multiplexed) {
+    return unless $header && all { $_ } @$header;
+  } else {
+    return unless $header;
+  }

 
   # Special case: human stupidity
   # people insist that case sensitivity doesn't exist and try to enter all
   # sorts of stuff. at this point we've got a profile (with keys that represent
   # valid methods), and a header full of strings. if two of them match, the user
 
   # Special case: human stupidity
   # people insist that case sensitivity doesn't exist and try to enter all
   # sorts of stuff. at this point we've got a profile (with keys that represent
   # valid methods), and a header full of strings. if two of them match, the user

-  # mopst likely meant that field, so rewrite the header
+  # most likely meant that field, so rewrite the header

   if ($self->case_insensitive_header) {
     die 'case_insensitive_header is only possible with profile' unless $self->profile;
     if ($header) {
   if ($self->case_insensitive_header) {
     die 'case_insensitive_header is only possible with profile' unless $self->profile;
     if ($header) {

-      my $p_num = 0;
-      foreach my $h (@{ $header }) {
-        my @names = (
-          keys %{ $self->profile->[$p_num]->{profile} || {} },
+      my $h_aref = ($self->is_multiplexed)? $header : [ $header ];
+      my $p_num  = 0;
+      foreach my $h (@{ $h_aref }) {
+        my %names = (
+          (map { $_ => $_                                     } keys %{ $self->profile->[$p_num]->{profile} || {} }),
+          (map { $_ => $self->profile->[$p_num]{mapping}{$_}  } keys %{ $self->profile->[$p_num]->{mapping} || {} }),

         );
         );

-        for my $name (@names) {
+        for my $name (keys %names) {

           for my $i (0..$#$h) {
           for my $i (0..$#$h) {

-            $h->[$i] = $name if lc $h->[$i] eq lc $name;
+            $h->[$i] = $names{$name} if lc $h->[$i] eq lc $name;

           }
         }
         $p_num++;
           }
         }
         $p_num++;


@@ -206,6 +219,25 @@ sub _check_header {
   return $self->header($header);
 }
 
   return $self->header($header);
 }
 

+sub _check_multiplex_datatype_position {
+  my ($self) = @_;
+
+  return 1 if !$self->is_multiplexed; # ok if not multiplexed
+
+  my @positions = map { firstidx { 'datatype' eq lc($_) } @{ $_ } } @{ $self->header };
+  my $first_pos = $positions[0];
+  if (all { $first_pos == $_ } @positions) {
+    $self->_multiplex_datatype_position($first_pos);
+    return 1;
+  } else {
+    $self->_push_error([0,
+                        "datatype field must be at the same position for all datatypes for multiplexed data",
+                        0,
+                        0]);
+    return 0;
+  }
+}
+

 sub _parse_data {
   my ($self, %params) = @_;
   my (@data, @errors);
 sub _parse_data {
   my ($self, %params) = @_;
   my (@data, @errors);


@@ -214,6 +246,14 @@ sub _parse_data {
     my $row = $self->_csv->getline($self->_io);
     if ($row) {
       my $header = $self->_header_by_row($row);
     my $row = $self->_csv->getline($self->_io);
     if ($row) {
       my $header = $self->_header_by_row($row);

+      if (!$header) {
+        push @errors, [
+          0,
+          "Cannot get header for row. Maybe row name and datatype field not matching.",
+          0,
+          0];
+        last;
+      }

       my %hr;
       @hr{@{ $header }} = @$row;
       push @data, \%hr;
       my %hr;
       @hr{@{ $header }} = @$row;
       push @data, \%hr;


@@ -251,9 +291,9 @@ sub _header_by_row {
   }
 
   if ($self->is_multiplexed) {
   }
 
   if ($self->is_multiplexed) {

-    return $self->_row_header->{$row->[0]}
+    return $self->_row_header->{$row->[$self->_multiplex_datatype_position]}

   } else {
   } else {

-    return $self->header->[0];
+    return $self->header;

   }
 }
 
   }
 }
 


@@ -301,6 +341,9 @@ sub _push_error {
   $self->_errors(\@new_errors);
 }
 
   $self->_errors(\@new_errors);
 }
 

+sub specs {
+  $_[0]->dispatcher->_specs
+}

 
 1;
 
 
 1;
 


@@ -322,14 +365,14 @@ SL::Helper::Csv - take care of csv file uploads
     sep_char    => ',',     # default ';'
     quote_char  => '\'',    # default '"'
     escape_char => '"',     # default '"'
     sep_char    => ',',     # default ';'
     quote_char  => '\'',    # default '"'
     escape_char => '"',     # default '"'

-    header      => [ [qw(id text sellprice word)] ], # see later
+    header      => [ qw(id text sellprice word) ], # see later

     profile     => [ { profile => { sellprice => 'sellprice_as_number'},
                        class   => 'SL::DB::Part' } ],
   );
 
   my $status  = $csv->parse;
   my $hrefs   = $csv->get_data;
     profile     => [ { profile => { sellprice => 'sellprice_as_number'},
                        class   => 'SL::DB::Part' } ],
   );
 
   my $status  = $csv->parse;
   my $hrefs   = $csv->get_data;

-  my @objects = $csv->get_objects;
+  my $objects = $csv->get_objects;

 
   my @errors  = $csv->errors;
 
 
   my @errors  = $csv->errors;
 


@@ -337,8 +380,8 @@ SL::Helper::Csv - take care of csv file uploads
 
 See Synopsis.
 
 
 See Synopsis.
 

-Text::CSV offeres already good functions to get lines out of a csv file, but in
-most cases you will want those line to be parsed into hashes or even objects,
+Text::CSV already offers good functions to get lines out of a csv file, but in
+most cases you will want those lines to be parsed into hashes or even objects,

 so this model just skips ahead and gives you objects.
 
 Its basic assumptions are:
 so this model just skips ahead and gives you objects.
 
 Its basic assumptions are:


@@ -347,7 +390,7 @@ Its basic assumptions are:
 
 =item You do know what you expect to be in that csv file.
 
 
 =item You do know what you expect to be in that csv file.
 

-This means first and foremost you have knowledge about encoding, number and
+This means first and foremost that you have knowledge about encoding, number and

 date format, csv parameters such as quoting and separation characters. You also
 know what content will be in that csv and what L<Rose::DB> is responsible for
 it. You provide valid header columns and their mapping to the objects.
 date format, csv parameters such as quoting and separation characters. You also
 know what content will be in that csv and what L<Rose::DB> is responsible for
 it. You provide valid header columns and their mapping to the objects.


@@ -370,7 +413,7 @@ nothing more.
 This module can handle multiplexed data of different class types. In that case
 multiple profiles with classes and row identifiers must be given. Multiple
 headers may also be given or read from csv data. Data must contain the row
 This module can handle multiplexed data of different class types. In that case
 multiple profiles with classes and row identifiers must be given. Multiple
 headers may also be given or read from csv data. Data must contain the row

-identifier in the first column and it's field name must be 'datatype'.
+identifier in the column named 'datatype'.

 
 =back
 
 
 =back
 


@@ -390,7 +433,7 @@ Do the actual work. Will return true ($self actually) if success, undef if not.
 
 Parse the data into objects and return those.
 
 
 Parse the data into objects and return those.
 

-This method will return list or arrayref depending on context.
+This method will return an arrayref of all objects.

 
 =item C<get_data>
 
 
 =item C<get_data>
 


@@ -427,99 +470,154 @@ Same as in L<Text::CSV>
 
 =item C<header> \@HEADERS
 
 
 =item C<header> \@HEADERS
 

-If given, it contains an ARRAYREF for each different class type (i.e. one
-ARRAYREF if the data is only of one class type). These ARRAYREFS are the header
-fields which are an array of columns. In this case the first lines are not used
-as a header. Empty header fields will be ignored in objects.
+If given, it contains an ARRAY of the header fields for not multiplexed data.
+Or an ARRAYREF for each different class type for multiplexed data. These
+ARRAYREFS are the header fields which are an array of columns. In this case
+the first lines are not used as a header. Empty header fields will be ignored
+in objects.

 
 If not given, headers are taken from the first n lines of data, where n is the
 number of different class types.
 
 
 If not given, headers are taken from the first n lines of data, where n is the
 number of different class types.
 

+In case of multiplexed data there must be a column named 'datatype'. This
+column must be given in each header and must be at the same position in each
+header.
+

 Examples:
 
   classic data of one type:
 Examples:
 
   classic data of one type:

-  [ [ 'name', 'street', 'zipcode', 'city' ] ]
+  [ 'name', 'street', 'zipcode', 'city' ]
+
+  multiplexed data with two different types:
+  [ [ 'datatype', 'ordernumber', 'customer', 'transdate' ],
+    [ 'datatype', 'partnumber', 'qty', 'sellprice' ] ]
+
+=item C<profile> PROFILE_DATA
+
+The profile mapping csv to the objects.

 
 

-  multiplexed data with two different types
-  [ [ 'ordernumber', 'customer', 'transdate' ], [ 'partnumber', 'qty', 'sellprice' ] ]
+See section L</PROFILE> for information on this topic.

 
 

-=item C<profile> [{profile => \%ACCESSORS, class => class, row_ident => ri},]
+=item C<ignore_unknown_columns>
+
+If set, the import will ignore unkown header columns. Useful for lazy imports,
+but deactivated by default.
+
+=item C<case_insensitive_header>
+
+If set, header columns will be matched against profile entries case
+insensitive, and on match the profile name will be taken.
+
+Only works if a profile is given, will die otherwise.

 
 

-This is an ARRAYREF to HASHREFs which may contain the keys C<profile>, C<class>
-and C<row_ident>.
+If both C<case_insensitive_header> and C<strict_profile> is set, matched header
+columns will be accepted.
+
+=item C<strict_profile>
+
+If set, all columns to be parsed must be specified in C<profile>. Every header
+field not listed there will be treated like an unknown column.
+
+If both C<case_insensitive_header> and C<strict_profile> is set, matched header
+columns will be accepted.

 
 

-The C<profile> is a HASHREF which may be used to map header fields to custom
+=back
+
+=head1 PROFILE
+
+The profile is needed for mapping csv data to the accessors in the data object.
+
+The basic structure is:
+
+  PROFILE       := [ CLASS_PROFILE, CLASS_PROFILE* ]
+  CLASS_PROFILE := {
+                      profile   => { ACCESSORS+ },
+                      class     => $classname,
+                      row_ident => $row_ident,
+                      mapping   => { MAPPINGS* },
+                   }
+  ACCESSORS     := $field => $accessor
+  MAPPINGS      := $alias => $field
+
+The C<ACCESSORS> may be used to map header fields to custom

 accessors. Example:
 
 accessors. Example:
 

-  [ {profile => { listprice => listprice_as_number }} ]
+  profile => {
+    listprice => 'listprice_as_number',
+  }

 
 

-In this case C<listprice_as_number> will be used to read in values from the
+In this case C<listprice_as_number> will be used to store the values from the

 C<listprice> column.
 
 C<listprice> column.
 

-In case of a One-To-One relationsship these can also be set over
-relationsships by sparating the steps with a dot (C<.>). This will work:
+In case of a One-To-One relationship these can also be set over
+relationships by separating the steps with a dot (C<.>). This will work:

 
 

-  [ {profile => { customer => 'customer.name' }} ]
+  customer => 'customer.name',

 
 And will result in something like this:
 
   $obj->customer($obj->meta->relationship('customer')->class->new);
   $obj->customer->name($csv_line->{customer})
 
 
 And will result in something like this:
 
   $obj->customer($obj->meta->relationship('customer')->class->new);
   $obj->customer->name($csv_line->{customer})
 

-But beware, this will not try to look up anything in the database. You will
+Beware, this will not try to look up anything in the database! You will

 simply receive objects that represent what the profile defined. If some of
 simply receive objects that represent what the profile defined. If some of

-these information are unique, and should be connected to preexisting data, you
+these information are unique, or should be connected to preexisting data, you

 will have to do that for yourself. Since you provided the profile, it is
 assumed you know what to do in this case.
 
 If no profile is given, any header field found will be taken as is.
 
 If the path in a profile entry is empty, the field will be subjected to
 will have to do that for yourself. Since you provided the profile, it is
 assumed you know what to do in this case.
 
 If no profile is given, any header field found will be taken as is.
 
 If the path in a profile entry is empty, the field will be subjected to

-C<strict_profile> and C<case_insensitive_header> checking, will be parsed into
-C<get_data>, but will not be attempted to be dispatched into objects.
+C<strict_profile> and C<case_insensitive_header> checking and will be parsed
+into C<get_data>, but will not be attempted to be dispatched into objects.

 
 

-If C<class> is present, the line will be handed to the new sub of this class,
-and the return value used instead of the line itself.
+C<class> must be present. A new instance will be created for each line before
+dispatching into it.

 
 

-C<row_ident> is a string to recognize the right profile and class for each data
-line in multiplexed data.
+C<row_ident> is used to determine the correct profile in multiplexed data and
+must be given there. It's not used in non-multiplexed data.

 
 

-In case of multiplexed data, C<class> and C<row_ident> must be given.
-Example:
-  [ {
-      class     => 'SL::DB::Order',
-      row_ident => 'O'
-    },
-    {
-      class     => 'SL::DB::OrderItem',
-      row_ident => 'I',
-      profile   => {sellprice => sellprice_as_number}
-    } ]
+If C<mappings> is present, it must contain a hashref that maps strings to known
+fields. This can be used to add custom profiles for known sources, that don't
+comply with the expected header identities.

 
 

-=item C<ignore_unknown_columns>
-
-If set, the import will ignore unkown header columns. Useful for lazy imports,
-but deactivated by default.
+Without strict profiles, mappings can also directly map header fields that
+should end up in the same accessor.

 
 

-=item C<case_insensitive_header>
+With case insensitive headings, mappings will also modify the headers, to fit
+the expected profile.

 
 

-If set, header columns will be matched against profile entries case
-insensitive, and on match the profile name will be taken.
+Mappings can be identical to known fields and will be prefered during lookup,
+but will not replace the field, meaning that:

 
 

-Only works if a profile is given, will die otherwise.
-
-If both C<case_insensitive_header> and C<strict_profile> is set, matched header
-columns will be accepted.
+  profile => {
+    name        => 'name',
+    description => 'description',
+  }
+  mapping => {
+    name        => 'description',
+    shortname   => 'name',
+  }

 
 

-=item C<strict_profile>
+will work as expected, and shortname will not end up in description. This also
+works with the case insensitive option. Note however that the case insensitive
+option will not enable true unicode collating.

 
 

-If set, all columns to be parsed must be specified in C<profile>. Every header
-field not listed there will be treated like an unknown column.

 
 

-If both C<case_insensitive_header> and C<strict_profile> is set, matched header
-columns will be accepted.
+Here's a full example:

 
 

-=back
+  [
+    {
+      class     => 'SL::DB::Order',
+      row_ident => 'O'
+    },
+    {
+      class     => 'SL::DB::OrderItem',
+      row_ident => 'I',
+      profile   => { sellprice => 'sellprice_as_number' },
+      mapping   => { 'Verkaufspreis' => 'sellprice' }
+    },
+  ]

 
 =head1 ERROR HANDLING
 
 
 =head1 ERROR HANDLING
 


@@ -534,6 +632,9 @@ Each entry is an object with the following attributes:
 
 Note that the last entry can be off, but will give an estimate.
 
 
 Note that the last entry can be off, but will give an estimate.
 

+Error handling is also known to break on new Perl versions and need to be
+adjusted from time to time due to changes in Text::CSV_XS.
+

 =head1 CAVEATS
 
 =over 4
 =head1 CAVEATS
 
 =over 4