1 package SL::Helper::Csv;
9 use Params::Validate qw(:all);
11 use Rose::Object::MakeMethods::Generic scalar => [ qw(
12 file encoding sep_char quote_char escape_char header profile
13 numberformat dateformat ignore_unknown_columns strict_profile _io _csv
14 _objects _parsed _data _errors all_cvar_configs case_insensitive_header
17 use SL::Helper::Csv::Dispatcher;
18 use SL::Helper::Csv::Error;
24 my %params = validate(@_, {
25 sep_char => { default => ';' },
26 quote_char => { default => '"' },
27 escape_char => { default => '"' },
28 header => { type => ARRAYREF, optional => 1 },
29 profile => { type => HASHREF, optional => 1 },
34 ignore_unknown_columns => 0,
36 case_insensitive_header => 0,
38 my $self = bless {}, $class;
40 $self->$_($params{$_}) for keys %params;
42 $self->_io(IO::File->new);
43 $self->_csv(Text::CSV_XS->new({
45 sep_char => $self->sep_char,
46 quote_char => $self->quote_char,
47 escape_char => $self->escape_char,
56 my ($self, %params) = @_;
59 return if ! $self->_check_header;
60 return if ! $self->dispatcher->parse_profile;
61 return if ! $self->_parse_data;
72 my ($self, %params) = @_;
73 croak 'must parse first' unless $self->_parsed;
75 $self->_make_objects unless $self->_objects;
76 return wantarray ? @{ $self->_objects } : $self->_objects;
90 my ($self, %params) = @_;
92 $self->encoding($self->_guess_encoding) if !$self->encoding;
94 $self->_io->open($self->file, '<' . $self->_encode_layer)
95 or die "could not open file " . $self->file;
101 my ($self, %params) = @_;
102 my $header = $self->header;
105 $header = $self->_csv->getline($self->_io);
108 $self->_csv->error_input,
109 $self->_csv->error_diag,
114 # Special case: utf8 BOM.
115 # certain software (namely MS Office and notepad.exe insist on prefixing
116 # data with a discouraged but valid byte order mark
117 # if not removed, the first header field will not be recognized
118 if ($header && $header->[0] && $self->encoding =~ /utf-?8/i) {
119 $header->[0] =~ s/^\x{FEFF}//;
122 return unless $header;
124 # Special case: human stupidity
125 # people insist that case sensitivity doesn't exist and try to enter all
126 # sorts of stuff. at this point we've got a profile (with keys that represent
127 # valid methods), and a header full of strings. if two of them match, the user
128 # mopst likely meant that field, so rewrite the header
129 if ($self->case_insensitive_header) {
130 die 'case_insensitive_header is only possible with profile' unless $self->profile;
132 keys %{ $self->profile || {} },
134 for my $name (@names) {
135 for my $i (0..$#$header) {
136 $header->[$i] = $name if lc $header->[$i] eq lc $name;
141 return $self->header($header);
145 my ($self, %params) = @_;
148 $self->_csv->column_names(@{ $self->header });
151 my $row = $self->_csv->getline($self->_io);
154 @hr{@{ $self->header }} = @$row;
157 last if $self->_csv->eof;
158 # Text::CSV_XS 0.89 added record number to error_diag
159 if (qv(Text::CSV_XS->VERSION) >= qv('0.89')) {
161 $self->_csv->error_input,
162 $self->_csv->error_diag,
166 $self->_csv->error_input,
167 $self->_csv->error_diag,
168 $self->_io->input_line_number,
172 last if $self->_csv->eof;
175 $self->_data(\@data);
176 $self->_push_error(@errors);
182 ':encoding(' . $_[0]->encoding . ')';
186 my ($self, %params) = @_;
189 local $::myconfig{numberformat} = $self->numberformat if $self->numberformat;
190 local $::myconfig{dateformat} = $self->dateformat if $self->dateformat;
192 for my $line (@{ $self->_data }) {
193 my $tmp_obj = $self->dispatcher->dispatch($line);
194 push @objs, $tmp_obj;
197 $self->_objects(\@objs);
201 my ($self, %params) = @_;
203 $self->{_dispatcher} ||= $self->_make_dispatcher;
206 sub _make_dispatcher {
207 my ($self, %params) = @_;
209 die 'need a header to make a dispatcher' unless $self->header;
211 return SL::Helper::Csv::Dispatcher->new($self);
214 sub _guess_encoding {
220 my ($self, @errors) = @_;
221 my @new_errors = ($self->errors, map { SL::Helper::Csv::Error->new(@$_) } @errors);
222 $self->_errors(\@new_errors);
234 SL::Helper::Csv - take care of csv file uploads
240 my $csv = SL::Helper::Csv->new(
241 file => \$::form->{upload_file},
242 encoding => 'utf-8', # undef means utf8
243 sep_char => ',', # default ';'
244 quote_char => '\'', # default '"'
245 escape_char => '"', # default '"'
246 header => [qw(id text sellprice word)], # see later
247 profile => { profile => { sellprice => 'sellprice_as_number'}, class => SL::DB::Part },
250 my $status = $csv->parse;
251 my $hrefs = $csv->get_data;
252 my @objects = $csv->get_objects;
254 my @errors = $csv->errors;
260 Text::CSV offeres already good functions to get lines out of a csv file, but in
261 most cases you will want those line to be parsed into hashes or even objects,
262 so this model just skips ahead and gives you objects.
264 Its basic assumptions are:
268 =item You do know what you expect to be in that csv file.
270 This means first and foremost you have knowledge about encoding, number and
271 date format, csv parameters such as quoting and separation characters. You also
272 know what content will be in that csv and what L<Rose::DB> is responsible for
273 it. You provide valid header columns and their mapping to the objects.
275 =item You do NOT know if the csv provider yields to your expectations.
277 Stuff that does not work with what you expect should not crash anything, but
278 give you a hint what went wrong. As a result, if you remember to check for
279 errors after each step, you should be fine.
281 =item Data does not make sense. It's just data.
283 Almost all data imports have some type of constraints. Some data needs to be
284 unique, other data needs to be connected to existing data sets. This will not
285 happen here. You will receive a plain mapping of the data into the class tree,
296 Standard constructor. You can use this to set most of the data.
300 Do the actual work. Will return true ($self actually) if success, undef if not.
304 Parse the data into objects and return those.
306 This method will return list or arrayref depending on context.
310 Returns an arrayref of the raw lines as hashrefs.
314 Return all errors that came up during parsing. See error handling for detailed
325 The file which contents are to be read. Can be a name of a physical file or a
326 scalar ref for memory data.
330 Encoding of the CSV file. Note that this module does not do any encoding
331 guessing. Know what your data is. Defaults to utf-8.
339 Same as in L<Text::CSV>
341 =item C<header> \@FIELDS
343 Can be an array of columns, in this case the first line is not used as a
344 header. Empty header fields will be ignored in objects.
346 =item C<profile> {profile => \%ACCESSORS, class => class}
348 This is a HASHREF to hash which may contain the keys C<profile> and C<class>.
350 The C<profile> is a HASHREF which may be used to map header fields to custom
353 {profile => { listprice => listprice_as_number }}
355 In this case C<listprice_as_number> will be used to read in values from the
358 In case of a One-To-One relationsship these can also be set over
359 relationsships by sparating the steps with a dot (C<.>). This will work:
361 {profile => { customer => 'customer.name' }}
363 And will result in something like this:
365 $obj->customer($obj->meta->relationship('customer')->class->new);
366 $obj->customer->name($csv_line->{customer})
368 But beware, this will not try to look up anything in the database. You will
369 simply receive objects that represent what the profile defined. If some of
370 these information are unique, and should be connected to preexisting data, you
371 will have to do that for yourself. Since you provided the profile, it is
372 assumed you know what to do in this case.
374 If C<class> is present, the line will be handed to the new sub of this class,
375 and the return value used instead of the line itself.
377 =item C<ignore_unknown_columns>
379 If set, the import will ignore unkown header columns. Useful for lazy imports,
380 but deactivated by default.
382 =item C<case_insensitive_header>
384 If set, header columns will be matched against profile entries case
385 insensitive, and on match the profile name will be taken.
387 Only works if a profile is given, will die otherwise.
389 If both C<case_insensitive_header> and C<strict_profile> is set, matched header
390 columns will be accepted.
392 =item C<strict_profile>
394 If set, all columns to be parsed must be specified in C<profile>. Every header
395 field not listed there will be treated like an unknown column.
397 If both C<case_insensitive_header> and C<strict_profile> is set, matched header
398 columns will be accepted.
402 =head1 ERROR HANDLING
404 After parsing a file all errors will be accumulated into C<errors>.
405 Each entry is an object with the following attributes:
407 raw_input: offending raw input,
408 code: Text::CSV error code if Text:CSV signalled an error, 0 else,
409 diag: error diagnostics,
410 line: position in line,
411 col: estimated line in file,
413 Note that the last entry can be off, but will give an estimate.
421 sep_char, quote_char, and escape_char are passed to Text::CSV on creation.
422 Changing them later has no effect currently.
426 Encoding errors are not dealt with properly.
432 Dispatch to child objects, like this:
434 $csv = SL::Helper::Csv->new(
447 class => SL::DB::Part,
453 Sven Schöling E<lt>s.schoeling@linet-services.deE<gt>