use Carp;
use IO::File;
use Params::Validate qw(:all);
-use Text::CSV;
+use Text::CSV_XS;
use Rose::Object::MakeMethods::Generic scalar => [ qw(
file encoding sep_char quote_char escape_char header profile class
- numberformat dateformat ignore_unknown_columns _io _csv _objects _parsed
- _data _errors
+ numberformat dateformat ignore_unknown_columns strict_profile _io _csv
+ _objects _parsed _data _errors all_cvar_configs case_insensitive_header
) ];
use SL::Helper::Csv::Dispatcher;
+use SL::Helper::Csv::Error;
# public interface
numberformat => 0,
dateformat => 0,
ignore_unknown_columns => 0,
+ strict_profile => 0,
+ case_insensitive_header => 0,
});
my $self = bless {}, $class;
$self->$_($params{$_}) for keys %params;
$self->_io(IO::File->new);
- $self->_csv(Text::CSV->new({
+ $self->_csv(Text::CSV_XS->new({
binary => 1,
sep_char => $self->sep_char,
quote_char => $self->quote_char,
sub _check_header {
my ($self, %params) = @_;
- return $self->header if $self->header;
+ my $header = $self->header;
- my $header = $self->_csv->getline($self->_io);
+ if (! $header) {
+ $header = $self->_csv->getline($self->_io);
- $self->_push_error([
- $self->_csv->error_input,
- $self->_csv->error_diag,
- 0,
- ]) unless $header;
+ $self->_push_error([
+ $self->_csv->error_input,
+ $self->_csv->error_diag,
+ 0,
+ ]) unless $header;
+ }
+
+ # Special case: utf8 BOM.
+ # certain software (namely MS Office and notepad.exe insist on prefixing
+ # data with a discouraged but valid byte order mark
+ # if not removed, the first header field will not be recognized
+ if ($header && $header->[0] && $self->encoding =~ /utf-?8/i) {
+ $header->[0] =~ s/^\x{FEFF}//;
+ }
+
+ return unless $header;
+
+ # Special case: human stupidity
+ # people insist that case sensitivity doesn't exist and try to enter all
+ # sorts of stuff. at this point we've got a profile (with keys that represent
+ # valid methods), and a header full of strings. if two of them match, the user
+ # mopst likely meant that field, so rewrite the header
+ if ($self->case_insensitive_header) {
+ die 'case_insensitive_header is only possible with profile' unless $self->profile;
+ my @names = (
+ keys %{ $self->profile || {} },
+ );
+ for my $name (@names) {
+ for my $i (0..$#$header) {
+ $header->[$i] = $name if lc $header->[$i] eq lc $name;
+ }
+ }
+ }
- $self->header($header);
+ return $self->header($header);
}
sub _parse_data {
while (1) {
my $row = $self->_csv->getline($self->_io);
- last if $self->_csv->eof;
if ($row) {
my %hr;
@hr{@{ $self->header }} = @$row;
push @data, \%hr;
} else {
+ last if $self->_csv->eof;
push @errors, [
$self->_csv->error_input,
$self->_csv->error_diag,
$self->_io->input_line_number,
];
}
+ last if $self->_csv->eof;
}
$self->_data(\@data);
sub _push_error {
my ($self, @errors) = @_;
- my @new_errors = ($self->errors, @errors);
+ my @new_errors = ($self->errors, map { SL::Helper::Csv::Error->new(@$_) } @errors);
$self->_errors(\@new_errors);
}
my $status = $csv->parse;
my $hrefs = $csv->get_data;
- my @objects = $scv->get_objects;
+ my @objects = $csv->get_objects;
+
+ my @errors = $csv->errors;
=head1 DESCRIPTION
most cases you will want those line to be parsed into hashes or even objects,
so this model just skips ahead and gives you objects.
-Encoding autodetection is not easy, and should not be trusted. Try to avoid it
-if possible.
+Its basic assumptions are:
+
+=over 4
+
+=item You do know what you expect to be in that csv file.
+
+This means first and foremost you have knowledge about encoding, number and
+date format, csv parameters such as quoting and separation characters. You also
+know what content will be in that csv and what L<Rose::DB> is responsible for
+it. You provide valid header columns and their mapping to the objects.
+
+=item You do NOT know if the csv provider yields to your expectations.
+
+Stuff that does not work with what you expect should not crash anything, but
+give you a hint what went wrong. As a result, if you remeber to check for
+errors after each step, you should be fine.
+
+=item Data does not make sense. It's just data.
+
+Almost all data imports have some type of constraints. Some data needs to be
+unique, other data needs to be connected to existing data sets. This will not
+happen here. You will receive a plain mapping of the data into the class tree,
+nothing more.
+
+=back
=head1 METHODS
=item C<errors>
-Return all errors that came up druing parsing. See error handling for detailed
+Return all errors that came up during parsing. See error handling for detailed
information.
=back
will have to do that for yourself. Since you provided the profile, it is
assumed you know what to do in this case.
+If no profile is given, any header field found will be taken as is.
+
+If the path in a profile entry is empty, the field will be subjected to
+C<strict_profile> and C<case_insensitive_header> checking, will be parsed into
+C<get_data>, but will not be attempted to be dispatched into objects.
+
=item C<class>
If present, the line will be handed to the new sub of this class,
If set, the import will ignore unkown header columns. Useful for lazy imports,
but deactivated by default.
+=item C<case_insensitive_header>
+
+If set, header columns will be matched against profile entries case
+insensitive, and on match the profile name will be taken.
+
+Only works if a profile is given, will die otherwise.
+
+If both C<case_insensitive_header> and C<strict_profile> is set, matched header
+columns will be accepted.
+
+=item C<strict_profile>
+
+If set, all columns to be parsed must be specified in C<profile>. Every header
+field not listed there will be treated like an unknown column.
+
+If both C<case_insensitive_header> and C<strict_profile> is set, matched header
+columns will be accepted.
+
=back
=head1 ERROR HANDLING
After parsing a file all errors will be accumulated into C<errors>.
+Each entry is an object with the following attributes:
-Each entry is an arrayref with the following structure:
-
- [
- 0 offending raw input,
- 1 Text::CSV error code if Text:CSV signalled an error, 0 else,
- 2 error diagnostics,
- 3 position in line,
- 4 estimated line in file,
- ]
+ raw_input: offending raw input,
+ code: Text::CSV error code if Text:CSV signalled an error, 0 else,
+ diag: error diagnostics,
+ line: position in line,
+ col: estimated line in file,
Note that the last entry can be off, but will give an estimate.
projects / kivitendo-erp.git / blobdiff