1 package SL::Helper::Csv;
8 use Params::Validate qw(:all);
10 use Rose::Object::MakeMethods::Generic scalar => [ qw(
11 file encoding sep_char quote_char escape_char header profile class
12 numberformat dateformat ignore_unknown_columns _io _csv _objects _parsed
16 use SL::Helper::Csv::Dispatcher;
17 use SL::Helper::Csv::Error;
23 my %params = validate(@_, {
24 sep_char => { default => ';' },
25 quote_char => { default => '"' },
26 escape_char => { default => '"' },
27 header => { type => ARRAYREF, optional => 1 },
28 profile => { type => HASHREF, optional => 1 },
34 ignore_unknown_columns => 0,
36 my $self = bless {}, $class;
38 $self->$_($params{$_}) for keys %params;
40 $self->_io(IO::File->new);
41 $self->_csv(Text::CSV->new({
43 sep_char => $self->sep_char,
44 quote_char => $self->quote_char,
45 escape_char => $self->escape_char,
54 my ($self, %params) = @_;
57 return if ! $self->_check_header;
58 return if ! $self->dispatcher->parse_profile;
59 return if ! $self->_parse_data;
70 my ($self, %params) = @_;
71 croak 'no class given' unless $self->class;
72 croak 'must parse first' unless $self->_parsed;
74 $self->_make_objects unless $self->_objects;
75 return wantarray ? @{ $self->_objects } : $self->_objects;
89 my ($self, %params) = @_;
91 $self->encoding($self->_guess_encoding) if !$self->encoding;
93 $self->_io->open($self->file, '<' . $self->_encode_layer)
94 or die "could not open file " . $self->file;
100 my ($self, %params) = @_;
101 return $self->header if $self->header;
103 my $header = $self->_csv->getline($self->_io);
106 $self->_csv->error_input,
107 $self->_csv->error_diag,
111 $self->header($header);
115 my ($self, %params) = @_;
118 $self->_csv->column_names(@{ $self->header });
121 my $row = $self->_csv->getline($self->_io);
122 last if $self->_csv->eof;
125 @hr{@{ $self->header }} = @$row;
129 $self->_csv->error_input,
130 $self->_csv->error_diag,
131 $self->_io->input_line_number,
136 $self->_data(\@data);
137 $self->_push_error(@errors);
143 ':encoding(' . $_[0]->encoding . ')';
147 my ($self, %params) = @_;
150 eval "require " . $self->class;
151 local $::myconfig{numberformat} = $self->numberformat if $self->numberformat;
152 local $::myconfig{dateformat} = $self->dateformat if $self->dateformat;
154 for my $line (@{ $self->_data }) {
155 my $tmp_obj = $self->class->new;
156 $self->dispatcher->dispatch($tmp_obj, $line);
157 push @objs, $tmp_obj;
160 $self->_objects(\@objs);
164 my ($self, %params) = @_;
166 $self->{_dispatcher} ||= $self->_make_dispatcher;
169 sub _make_dispatcher {
170 my ($self, %params) = @_;
172 die 'need a header to make a dispatcher' unless $self->header;
174 return SL::Helper::Csv::Dispatcher->new($self);
177 sub _guess_encoding {
183 my ($self, @errors) = @_;
184 my @new_errors = ($self->errors, map { SL::Helper::Csv::Error->new(@$_) } @errors);
185 $self->_errors(\@new_errors);
197 SL::Helper::Csv - take care of csv file uploads
203 my $csv = SL::Helper::Csv->new(
204 file => \$::form->{upload_file},
205 encoding => 'utf-8', # undef means utf8
206 sep_char => ',', # default ';'
207 quote_char => '\'', # default '"'
208 escape_char => '"', # default '"'
209 header => [qw(id text sellprice word)], # see later
210 profile => { sellprice => 'sellprice_as_number' },
211 class => 'SL::DB::CsvLine', # if present, map lines to this
214 my $status = $csv->parse;
215 my $hrefs = $csv->get_data;
216 my @objects = $csv->get_objects;
218 my @errors = $csv->errors;
224 Text::CSV offeres already good functions to get lines out of a csv file, but in
225 most cases you will want those line to be parsed into hashes or even objects,
226 so this model just skips ahead and gives you objects.
228 Its basic assumptions are:
232 =item You do know what you expect to be in that csv file.
234 This means first and foremost you have knowledge about encoding, number and
235 date format, csv parameters such as quoting and separation characters. You also
236 know what content will be in that csv and what L<Rose::DB> is responsible for
237 it. You provide valid header columns and their mapping to the objects.
239 =item You do NOT know if the csv provider yields to your expectations.
241 Stuff that does not work with what you expect should not crash anything, but
242 give you a hint what went wrong. As a result, if you remeber to check for
243 errors after each step, you should be fine.
245 =item Data does not make sense. It's just data.
247 Almost all data imports have some type of constraints. Some data needs to be
248 unique, other data needs to be connected to existing data sets. This will not
249 happen here. You will receive a plain mapping of the data into the class tree,
260 Standard constructor. You can use this to set most of the data.
264 Do the actual work. Will return true ($self actually) if success, undef if not.
268 Parse the data into objects and return those.
270 This method will return list or arrayref depending on context.
274 Returns an arrayref of the raw lines as hashrefs.
278 Return all errors that came up during parsing. See error handling for detailed
289 The file which contents are to be read. Can be a name of a physical file or a
290 scalar ref for memory data.
294 Encoding of the CSV file. Note that this module does not do any encoding
295 guessing. Know what your data is. Defaults to utf-8.
303 Same as in L<Text::CSV>
305 =item C<header> \@FIELDS
307 Can be an array of columns, in this case the first line is not used as a
308 header. Empty header fields will be ignored in objects.
310 =item C<profile> \%ACCESSORS
312 May be used to map header fields to custom accessors. Example:
314 { listprice => listprice_as_number }
316 In this case C<listprice_as_number> will be used to read in values from the
319 In case of a One-To-One relationsship these can also be set over
320 relationsships by sparating the steps with a dot (C<.>). This will work:
322 { customer => 'customer.name' }
324 And will result in something like this:
326 $obj->customer($obj->meta->relationship('customer')->class->new);
327 $obj->customer->name($csv_line->{customer})
329 But beware, this will not try to look up anything in the database. You will
330 simply receive objects that represent what the profile defined. If some of
331 these information are unique, and should be connected to preexisting data, you
332 will have to do that for yourself. Since you provided the profile, it is
333 assumed you know what to do in this case.
337 If present, the line will be handed to the new sub of this class,
338 and the return value used instead of the line itself.
340 =item C<ignore_unknown_columns>
342 If set, the import will ignore unkown header columns. Useful for lazy imports,
343 but deactivated by default.
347 =head1 ERROR HANDLING
349 After parsing a file all errors will be accumulated into C<errors>.
350 Each entry is an object with the following attributes:
352 raw_input: offending raw input,
353 code: Text::CSV error code if Text:CSV signalled an error, 0 else,
354 diag: error diagnostics,
355 line: position in line,
356 col: estimated line in file,
358 Note that the last entry can be off, but will give an estimate.
366 sep_char, quote_char, and escape_char are passed to Text::CSV on creation.
367 Changing them later has no effect currently.
371 Encoding errors are not dealt with properly.
377 Dispatch to child objects, like this:
379 $csv = SL::Helper::Csv->new(
381 class => SL::DB::Part,
396 Sven Schöling E<lt>s.schoeling@linet-services.deE<gt>