X-Git-Url: http://wagnertech.de/git?a=blobdiff_plain;f=SL%2FController%2FCsvImport%2FBase.pm;h=db6dc4d87caae24b1631b2d1e1e816483ad140b2;hb=c6b2257945060625bcb86fa7e2efd27c737480ff;hp=f84d54b0e01421f125d3ad9533cfb1b0280d6afc;hpb=caf4380789ea2166eaf89f7a600406f850d64428;p=kivitendo-erp.git diff --git a/SL/Controller/CsvImport/Base.pm b/SL/Controller/CsvImport/Base.pm index f84d54b0e..db6dc4d87 100644 --- a/SL/Controller/CsvImport/Base.pm +++ b/SL/Controller/CsvImport/Base.pm @@ -5,6 +5,7 @@ use strict; use List::MoreUtils qw(pairwise); use SL::Helper::Csv; +use SL::DB::Currency; use SL::DB::Customer; use SL::DB::Language; use SL::DB::PaymentTerm; @@ -15,12 +16,16 @@ use parent qw(Rose::Object); use Rose::Object::MakeMethods::Generic ( - scalar => [ qw(controller file csv save_with_cascade) ], - 'scalar --get_set_init' => [ qw(profile displayable_columns existing_objects class manager_class cvar_columns all_cvar_configs all_languages payment_terms_by all_vc vc_by) ], + scalar => [ qw(controller file csv test_run save_with_cascade) ], + 'scalar --get_set_init' => [ qw(profile displayable_columns existing_objects class manager_class cvar_columns all_cvar_configs all_languages payment_terms_by all_currencies default_currency_id all_vc vc_by) ], ); sub run { - my ($self) = @_; + my ($self, %params) = @_; + + $self->test_run($params{test_run}); + + $self->controller->track_progress(phase => 'parsing csv', progress => 0); my $profile = $self->profile; $self->csv(SL::Helper::Csv->new(file => $self->file->file_name, @@ -29,14 +34,19 @@ sub run { profile => $profile, ignore_unknown_columns => 1, strict_profile => 1, + case_insensitive_header => 1, map { ( $_ => $self->controller->profile->get($_) ) } qw(sep_char escape_char quote_char), )); + $self->controller->track_progress(progress => 10); + my $old_numberformat = $::myconfig{numberformat}; $::myconfig{numberformat} = $self->controller->profile->get('numberformat'); $self->csv->parse; + $self->controller->track_progress(progress => 50); + $self->controller->errors([ $self->csv->errors ]) if $self->csv->errors; return if ( !$self->csv->header || $self->csv->errors ); @@ -49,9 +59,17 @@ sub run { $self->controller->info_headers({ used => { }, headers => [ ] }); my @objects = $self->csv->get_objects; + + $self->controller->track_progress(progress => 70); + my @raw_data = @{ $self->csv->get_data }; + + $self->controller->track_progress(progress => 80); + $self->controller->data([ pairwise { { object => $a, raw_data => $b, errors => [], information => [], info_data => {} } } @objects, @raw_data ]); + $self->controller->track_progress(progress => 90); + $self->check_objects; if ( $self->controller->profile->get('duplicates', 'no_check') ne 'no_check' ) { $self->check_std_duplicates(); @@ -59,6 +77,8 @@ sub run { } $self->fix_field_lengths; + $self->controller->track_progress(progress => 100); + $::myconfig{numberformat} = $old_numberformat; } @@ -103,6 +123,11 @@ sub add_cvar_raw_data_columns { map { $self->add_raw_data_columns($_) if exists $self->controller->data->[0]->{raw_data}->{$_} } @{ $self->cvar_columns }; } +sub init_all_cvar_configs { + # Must be overridden by derived specialized importer classes. + return []; +} + sub init_cvar_columns { my ($self) = @_; @@ -115,6 +140,18 @@ sub init_all_languages { return SL::DB::Manager::Language->get_all; } +sub init_all_currencies { + my ($self) = @_; + + return SL::DB::Manager::Currency->get_all; +} + +sub init_default_currency_id { + my ($self) = @_; + + return SL::DB::Default->get->currency_id; +} + sub init_payment_terms_by { my ($self) = @_; @@ -129,6 +166,10 @@ sub init_all_vc { vendors => SL::DB::Manager::Vendor->get_all }; } +sub force_allow_columns { + return (); +} + sub init_vc_by { my ($self) = @_; @@ -172,8 +213,6 @@ sub check_vc { sub handle_cvars { my ($self, $entry) = @_; - return unless $self->can('all_cvar_configs'); - my %type_to_column = ( text => 'text_value', textfield => 'text_value', select => 'text_value', @@ -200,6 +239,8 @@ sub init_profile { eval "require " . $self->class; my %unwanted = map { ( $_ => 1 ) } (qw(itime mtime), map { $_->name } @{ $self->class->meta->primary_key_columns }); + delete $unwanted{$_} for ($self->force_allow_columns); + my %profile; for my $col ($self->class->meta->columns) { next if $unwanted{$col}; @@ -212,7 +253,9 @@ sub init_profile { $profile{$col} = $name; } - $self->profile(\%profile); + $profile{ 'cvar_' . $_->name } = '' for @{ $self->all_cvar_configs }; + + \%profile; } sub add_displayable_columns { @@ -241,8 +284,6 @@ sub setup_displayable_columns { sub add_cvar_columns_to_displayable_columns { my ($self) = @_; - return unless $self->can('all_cvar_configs'); - $self->add_displayable_columns(map { { name => 'cvar_' . $_->name, description => $::locale->text('#1 (custom variable)', $_->description) } } @{ $self->all_cvar_configs }); @@ -357,7 +398,16 @@ sub save_objects { my $data = $params{data} || $self->controller->data; - foreach my $entry (@{ $data }) { + return unless $data->[0]; + return unless $data->[0]{object}; + + $self->controller->track_progress(phase => 'saving data', progress => 0); # scale from 45..95%; + + my $dbh = $data->[0]{object}->db; + + $dbh->begin_work; + foreach my $entry_index (0 .. $#$data) { + my $entry = $data->[$entry_index]; next if @{ $entry->{errors} }; my $object = $entry->{object_to_save} || $entry->{object}; @@ -367,7 +417,14 @@ sub save_objects { } else { $self->controller->num_imported($self->controller->num_imported + 1); } + } continue { + if ($entry_index % 100 == 0) { + $dbh->commit; + $self->controller->track_progress(progress => $entry_index/scalar(@$data) * 100); # scale from 45..95%; + $dbh->begin_work; + } } + $dbh->commit; } sub field_lengths { @@ -384,5 +441,20 @@ sub fix_field_lengths { } } -1; +sub clean_fields { + my ($self, $illegal_chars, $object, @fields) = @_; + + my @cleaned_fields; + foreach my $field (grep { $object->can($_) } @fields) { + my $value = $object->$field; + + next unless defined($value) && ($value =~ s/$illegal_chars/ /g); + $object->$field($value); + push @cleaned_fields, $field; + } + + return @cleaned_fields; +} + +1;