1 package SL::Helper::Csv;
9 use Params::Validate qw(:all);
11 use Rose::Object::MakeMethods::Generic scalar => [ qw(
12 file encoding sep_char quote_char escape_char header profile class
13 numberformat dateformat ignore_unknown_columns strict_profile _io _csv
14 _objects _parsed _data _errors all_cvar_configs case_insensitive_header
17 use SL::Helper::Csv::Dispatcher;
18 use SL::Helper::Csv::Error;
24 my %params = validate(@_, {
25 sep_char => { default => ';' },
26 quote_char => { default => '"' },
27 escape_char => { default => '"' },
28 header => { type => ARRAYREF, optional => 1 },
29 profile => { type => HASHREF, optional => 1 },
35 ignore_unknown_columns => 0,
37 case_insensitive_header => 0,
39 my $self = bless {}, $class;
41 $self->$_($params{$_}) for keys %params;
43 $self->_io(IO::File->new);
44 $self->_csv(Text::CSV_XS->new({
46 sep_char => $self->sep_char,
47 quote_char => $self->quote_char,
48 escape_char => $self->escape_char,
57 my ($self, %params) = @_;
60 return if ! $self->_check_header;
61 return if ! $self->dispatcher->parse_profile;
62 return if ! $self->_parse_data;
73 my ($self, %params) = @_;
74 croak 'no class given' unless $self->class;
75 croak 'must parse first' unless $self->_parsed;
77 $self->_make_objects unless $self->_objects;
78 return wantarray ? @{ $self->_objects } : $self->_objects;
92 my ($self, %params) = @_;
94 $self->encoding($self->_guess_encoding) if !$self->encoding;
96 $self->_io->open($self->file, '<' . $self->_encode_layer)
97 or die "could not open file " . $self->file;
103 my ($self, %params) = @_;
104 my $header = $self->header;
107 $header = $self->_csv->getline($self->_io);
110 $self->_csv->error_input,
111 $self->_csv->error_diag,
116 # Special case: utf8 BOM.
117 # certain software (namely MS Office and notepad.exe insist on prefixing
118 # data with a discouraged but valid byte order mark
119 # if not removed, the first header field will not be recognized
120 if ($header && $header->[0] && $self->encoding =~ /utf-?8/i) {
121 $header->[0] =~ s/^\x{FEFF}//;
124 return unless $header;
126 # Special case: human stupidity
127 # people insist that case sensitivity doesn't exist and try to enter all
128 # sorts of stuff. at this point we've got a profile (with keys that represent
129 # valid methods), and a header full of strings. if two of them match, the user
130 # mopst likely meant that field, so rewrite the header
131 if ($self->case_insensitive_header) {
132 die 'case_insensitive_header is only possible with profile' unless $self->profile;
134 keys %{ $self->profile || {} },
136 for my $name (@names) {
137 for my $i (0..$#$header) {
138 $header->[$i] = $name if lc $header->[$i] eq lc $name;
143 return $self->header($header);
147 my ($self, %params) = @_;
150 $self->_csv->column_names(@{ $self->header });
153 my $row = $self->_csv->getline($self->_io);
156 @hr{@{ $self->header }} = @$row;
159 last if $self->_csv->eof;
160 # Text::CSV_XS 0.89 added record number to error_diag
161 if (qv(Text::CSV_XS->VERSION) >= qv('0.89')) {
163 $self->_csv->error_input,
164 $self->_csv->error_diag,
168 $self->_csv->error_input,
169 $self->_csv->error_diag,
170 $self->_io->input_line_number,
174 last if $self->_csv->eof;
177 $self->_data(\@data);
178 $self->_push_error(@errors);
184 ':encoding(' . $_[0]->encoding . ')';
188 my ($self, %params) = @_;
191 eval "require " . $self->class;
192 local $::myconfig{numberformat} = $self->numberformat if $self->numberformat;
193 local $::myconfig{dateformat} = $self->dateformat if $self->dateformat;
195 for my $line (@{ $self->_data }) {
196 my $tmp_obj = $self->class->new;
197 $self->dispatcher->dispatch($tmp_obj, $line);
198 push @objs, $tmp_obj;
201 $self->_objects(\@objs);
205 my ($self, %params) = @_;
207 $self->{_dispatcher} ||= $self->_make_dispatcher;
210 sub _make_dispatcher {
211 my ($self, %params) = @_;
213 die 'need a header to make a dispatcher' unless $self->header;
215 return SL::Helper::Csv::Dispatcher->new($self);
218 sub _guess_encoding {
224 my ($self, @errors) = @_;
225 my @new_errors = ($self->errors, map { SL::Helper::Csv::Error->new(@$_) } @errors);
226 $self->_errors(\@new_errors);
238 SL::Helper::Csv - take care of csv file uploads
244 my $csv = SL::Helper::Csv->new(
245 file => \$::form->{upload_file},
246 encoding => 'utf-8', # undef means utf8
247 sep_char => ',', # default ';'
248 quote_char => '\'', # default '"'
249 escape_char => '"', # default '"'
250 header => [qw(id text sellprice word)], # see later
251 profile => { sellprice => 'sellprice_as_number' },
252 class => 'SL::DB::CsvLine', # if present, map lines to this
255 my $status = $csv->parse;
256 my $hrefs = $csv->get_data;
257 my @objects = $csv->get_objects;
259 my @errors = $csv->errors;
265 Text::CSV offeres already good functions to get lines out of a csv file, but in
266 most cases you will want those line to be parsed into hashes or even objects,
267 so this model just skips ahead and gives you objects.
269 Its basic assumptions are:
273 =item You do know what you expect to be in that csv file.
275 This means first and foremost you have knowledge about encoding, number and
276 date format, csv parameters such as quoting and separation characters. You also
277 know what content will be in that csv and what L<Rose::DB> is responsible for
278 it. You provide valid header columns and their mapping to the objects.
280 =item You do NOT know if the csv provider yields to your expectations.
282 Stuff that does not work with what you expect should not crash anything, but
283 give you a hint what went wrong. As a result, if you remeber to check for
284 errors after each step, you should be fine.
286 =item Data does not make sense. It's just data.
288 Almost all data imports have some type of constraints. Some data needs to be
289 unique, other data needs to be connected to existing data sets. This will not
290 happen here. You will receive a plain mapping of the data into the class tree,
301 Standard constructor. You can use this to set most of the data.
305 Do the actual work. Will return true ($self actually) if success, undef if not.
309 Parse the data into objects and return those.
311 This method will return list or arrayref depending on context.
315 Returns an arrayref of the raw lines as hashrefs.
319 Return all errors that came up during parsing. See error handling for detailed
330 The file which contents are to be read. Can be a name of a physical file or a
331 scalar ref for memory data.
335 Encoding of the CSV file. Note that this module does not do any encoding
336 guessing. Know what your data is. Defaults to utf-8.
344 Same as in L<Text::CSV>
346 =item C<header> \@FIELDS
348 Can be an array of columns, in this case the first line is not used as a
349 header. Empty header fields will be ignored in objects.
351 =item C<profile> \%ACCESSORS
353 May be used to map header fields to custom accessors. Example:
355 { listprice => listprice_as_number }
357 In this case C<listprice_as_number> will be used to read in values from the
360 In case of a One-To-One relationsship these can also be set over
361 relationsships by sparating the steps with a dot (C<.>). This will work:
363 { customer => 'customer.name' }
365 And will result in something like this:
367 $obj->customer($obj->meta->relationship('customer')->class->new);
368 $obj->customer->name($csv_line->{customer})
370 But beware, this will not try to look up anything in the database. You will
371 simply receive objects that represent what the profile defined. If some of
372 these information are unique, and should be connected to preexisting data, you
373 will have to do that for yourself. Since you provided the profile, it is
374 assumed you know what to do in this case.
376 If no profile is given, any header field found will be taken as is.
378 If the path in a profile entry is empty, the field will be subjected to
379 C<strict_profile> and C<case_insensitive_header> checking, will be parsed into
380 C<get_data>, but will not be attempted to be dispatched into objects.
384 If present, the line will be handed to the new sub of this class,
385 and the return value used instead of the line itself.
387 =item C<ignore_unknown_columns>
389 If set, the import will ignore unkown header columns. Useful for lazy imports,
390 but deactivated by default.
392 =item C<case_insensitive_header>
394 If set, header columns will be matched against profile entries case
395 insensitive, and on match the profile name will be taken.
397 Only works if a profile is given, will die otherwise.
399 If both C<case_insensitive_header> and C<strict_profile> is set, matched header
400 columns will be accepted.
402 =item C<strict_profile>
404 If set, all columns to be parsed must be specified in C<profile>. Every header
405 field not listed there will be treated like an unknown column.
407 If both C<case_insensitive_header> and C<strict_profile> is set, matched header
408 columns will be accepted.
412 =head1 ERROR HANDLING
414 After parsing a file all errors will be accumulated into C<errors>.
415 Each entry is an object with the following attributes:
417 raw_input: offending raw input,
418 code: Text::CSV error code if Text:CSV signalled an error, 0 else,
419 diag: error diagnostics,
420 line: position in line,
421 col: estimated line in file,
423 Note that the last entry can be off, but will give an estimate.
431 sep_char, quote_char, and escape_char are passed to Text::CSV on creation.
432 Changing them later has no effect currently.
436 Encoding errors are not dealt with properly.
442 Dispatch to child objects, like this:
444 $csv = SL::Helper::Csv->new(
446 class => SL::DB::Part,
461 Sven Schöling E<lt>s.schoeling@linet-services.deE<gt>