1 package SL::Helper::Csv;
8 use Params::Validate qw(:all);
10 use Rose::Object::MakeMethods::Generic scalar => [ qw(
11 file encoding sep_char quote_char escape_char header profile class
12 numberformat dateformat ignore_unknown_columns strict_profile _io _csv
13 _objects _parsed _data _errors
16 use SL::Helper::Csv::Dispatcher;
17 use SL::Helper::Csv::Error;
23 my %params = validate(@_, {
24 sep_char => { default => ';' },
25 quote_char => { default => '"' },
26 escape_char => { default => '"' },
27 header => { type => ARRAYREF, optional => 1 },
28 profile => { type => HASHREF, optional => 1 },
34 ignore_unknown_columns => 0,
37 my $self = bless {}, $class;
39 $self->$_($params{$_}) for keys %params;
41 $self->_io(IO::File->new);
42 $self->_csv(Text::CSV_XS->new({
44 sep_char => $self->sep_char,
45 quote_char => $self->quote_char,
46 escape_char => $self->escape_char,
55 my ($self, %params) = @_;
58 return if ! $self->_check_header;
59 return if ! $self->dispatcher->parse_profile;
60 return if ! $self->_parse_data;
71 my ($self, %params) = @_;
72 croak 'no class given' unless $self->class;
73 croak 'must parse first' unless $self->_parsed;
75 $self->_make_objects unless $self->_objects;
76 return wantarray ? @{ $self->_objects } : $self->_objects;
90 my ($self, %params) = @_;
92 $self->encoding($self->_guess_encoding) if !$self->encoding;
94 $self->_io->open($self->file, '<' . $self->_encode_layer)
95 or die "could not open file " . $self->file;
101 my ($self, %params) = @_;
102 my $header = $self->header;
105 $header = $self->_csv->getline($self->_io);
108 $self->_csv->error_input,
109 $self->_csv->error_diag,
114 # Special case: utf8 BOM.
115 # certain software (namely MS Office and notepad.exe insist on prefixing
116 # data with a discouraged but valid byte order mark
117 # if not removed, the first header field will not be recognized
118 if ($header && $header->[0] && $self->encoding =~ /utf-?8/i) {
119 $header->[0] =~ s/^\x{FEFF}//;
122 return unless $header;
123 return $self->header([ map { lc } @$header ]);
127 my ($self, %params) = @_;
130 $self->_csv->column_names(@{ $self->header });
133 my $row = $self->_csv->getline($self->_io);
136 @hr{@{ $self->header }} = @$row;
139 last if $self->_csv->eof;
141 $self->_csv->error_input,
142 $self->_csv->error_diag,
143 $self->_io->input_line_number,
146 last if $self->_csv->eof;
149 $self->_data(\@data);
150 $self->_push_error(@errors);
156 ':encoding(' . $_[0]->encoding . ')';
160 my ($self, %params) = @_;
163 eval "require " . $self->class;
164 local $::myconfig{numberformat} = $self->numberformat if $self->numberformat;
165 local $::myconfig{dateformat} = $self->dateformat if $self->dateformat;
167 for my $line (@{ $self->_data }) {
168 my $tmp_obj = $self->class->new;
169 $self->dispatcher->dispatch($tmp_obj, $line);
170 push @objs, $tmp_obj;
173 $self->_objects(\@objs);
177 my ($self, %params) = @_;
179 $self->{_dispatcher} ||= $self->_make_dispatcher;
182 sub _make_dispatcher {
183 my ($self, %params) = @_;
185 die 'need a header to make a dispatcher' unless $self->header;
187 return SL::Helper::Csv::Dispatcher->new($self);
190 sub _guess_encoding {
196 my ($self, @errors) = @_;
197 my @new_errors = ($self->errors, map { SL::Helper::Csv::Error->new(@$_) } @errors);
198 $self->_errors(\@new_errors);
210 SL::Helper::Csv - take care of csv file uploads
216 my $csv = SL::Helper::Csv->new(
217 file => \$::form->{upload_file},
218 encoding => 'utf-8', # undef means utf8
219 sep_char => ',', # default ';'
220 quote_char => '\'', # default '"'
221 escape_char => '"', # default '"'
222 header => [qw(id text sellprice word)], # see later
223 profile => { sellprice => 'sellprice_as_number' },
224 class => 'SL::DB::CsvLine', # if present, map lines to this
227 my $status = $csv->parse;
228 my $hrefs = $csv->get_data;
229 my @objects = $csv->get_objects;
231 my @errors = $csv->errors;
237 Text::CSV offeres already good functions to get lines out of a csv file, but in
238 most cases you will want those line to be parsed into hashes or even objects,
239 so this model just skips ahead and gives you objects.
241 Its basic assumptions are:
245 =item You do know what you expect to be in that csv file.
247 This means first and foremost you have knowledge about encoding, number and
248 date format, csv parameters such as quoting and separation characters. You also
249 know what content will be in that csv and what L<Rose::DB> is responsible for
250 it. You provide valid header columns and their mapping to the objects.
252 =item You do NOT know if the csv provider yields to your expectations.
254 Stuff that does not work with what you expect should not crash anything, but
255 give you a hint what went wrong. As a result, if you remeber to check for
256 errors after each step, you should be fine.
258 =item Data does not make sense. It's just data.
260 Almost all data imports have some type of constraints. Some data needs to be
261 unique, other data needs to be connected to existing data sets. This will not
262 happen here. You will receive a plain mapping of the data into the class tree,
273 Standard constructor. You can use this to set most of the data.
277 Do the actual work. Will return true ($self actually) if success, undef if not.
281 Parse the data into objects and return those.
283 This method will return list or arrayref depending on context.
287 Returns an arrayref of the raw lines as hashrefs.
291 Return all errors that came up during parsing. See error handling for detailed
302 The file which contents are to be read. Can be a name of a physical file or a
303 scalar ref for memory data.
307 Encoding of the CSV file. Note that this module does not do any encoding
308 guessing. Know what your data is. Defaults to utf-8.
316 Same as in L<Text::CSV>
318 =item C<header> \@FIELDS
320 Can be an array of columns, in this case the first line is not used as a
321 header. Empty header fields will be ignored in objects.
323 =item C<profile> \%ACCESSORS
325 May be used to map header fields to custom accessors. Example:
327 { listprice => listprice_as_number }
329 In this case C<listprice_as_number> will be used to read in values from the
332 In case of a One-To-One relationsship these can also be set over
333 relationsships by sparating the steps with a dot (C<.>). This will work:
335 { customer => 'customer.name' }
337 And will result in something like this:
339 $obj->customer($obj->meta->relationship('customer')->class->new);
340 $obj->customer->name($csv_line->{customer})
342 But beware, this will not try to look up anything in the database. You will
343 simply receive objects that represent what the profile defined. If some of
344 these information are unique, and should be connected to preexisting data, you
345 will have to do that for yourself. Since you provided the profile, it is
346 assumed you know what to do in this case.
350 If present, the line will be handed to the new sub of this class,
351 and the return value used instead of the line itself.
353 =item C<ignore_unknown_columns>
355 If set, the import will ignore unkown header columns. Useful for lazy imports,
356 but deactivated by default.
358 =item C<strict_profile>
360 If set, all columns to be parsed must be specified in C<profile>. Every header
361 field not listed there will be treated like an unknown column.
365 =head1 ERROR HANDLING
367 After parsing a file all errors will be accumulated into C<errors>.
368 Each entry is an object with the following attributes:
370 raw_input: offending raw input,
371 code: Text::CSV error code if Text:CSV signalled an error, 0 else,
372 diag: error diagnostics,
373 line: position in line,
374 col: estimated line in file,
376 Note that the last entry can be off, but will give an estimate.
384 sep_char, quote_char, and escape_char are passed to Text::CSV on creation.
385 Changing them later has no effect currently.
389 Encoding errors are not dealt with properly.
395 Dispatch to child objects, like this:
397 $csv = SL::Helper::Csv->new(
399 class => SL::DB::Part,
414 Sven Schöling E<lt>s.schoeling@linet-services.deE<gt>