From d74921657d967900bded4aaf4805647e9cf83562 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Sven=20Sch=C3=B6ling?= Date: Thu, 15 Nov 2012 15:16:26 +0100 Subject: [PATCH] Weitere Verbesserungen am asynchronen Import. - tracking - profile/session handling --- SL/BackgroundJob/CsvImport.pm | 36 ++++++++++++++++++-------- SL/Controller/CsvImport.pm | 45 ++++++++++++++++----------------- SL/Controller/CsvImport/Base.pm | 36 +++++++++++++++++++++++++- SL/Controller/CsvImport/Part.pm | 8 +++++- 4 files changed, 89 insertions(+), 36 deletions(-) diff --git a/SL/BackgroundJob/CsvImport.pm b/SL/BackgroundJob/CsvImport.pm index e34edc315..60147f8e2 100644 --- a/SL/BackgroundJob/CsvImport.pm +++ b/SL/BackgroundJob/CsvImport.pm @@ -5,6 +5,7 @@ use strict; use parent qw(SL::BackgroundJob::Base); use YAML (); +use SL::Controller::CsvImport; use SL::DB::CsvImportProfile; use SL::SessionFile::Random; @@ -15,12 +16,13 @@ sub create_job { $package =~ s/SL::BackgroundJob:://; my $profile = delete $params{profile} || SL::DB::CsvImportProfile->new; - my $result = delete $params{result} || SL::SessionFile::Random->new; + my $new_profile = $profile->clone_and_reset_deep; + $new_profile->save; my %data = ( - profile => { $profile->flatten }, - result => $result->file_name, %params, + profile_id => $new_profile->id, + session_id => $::auth->get_session_id, ); my $job = SL::DB::BackgroundJob->new( @@ -34,14 +36,11 @@ sub create_job { } sub profile { - my ($self, $db_obj) = @_; + my ($self) = @_; if (!$self->{profile}) { - $self->{profile} = SL::DB::CsvImportProfile->new; - my $data = YAML::Load($db_obj->data); - for (keys %$data) { - $self->{profile}->set($_ => $data->{$_}); - } + my $data = YAML::Load($self->{db_obj}->data); + $self->{profile} = SL::DB::Manager::CsvImportProfile->find_by(id => $data->{profile_id}); } return $self->{profile}; @@ -60,10 +59,25 @@ sub do_import { my ($self) = @_; my $c = SL::Controller::CsvImport->new; - $c->profile($self->{profile}); - $c->test_and_import(test => $self->); + $c->profile($self->profile); + $c->type($self->{db_obj}->data_as_hash->{type}); + $c->add_progress_tracker($self); + + $c->test_and_import(test => 1, session_id => $self->{db_obj}->data_as_hash->{session_id}); + + my $report_id = $c->save_report; + $self->{db_obj}->set_data(report_id => $report_id); + $self->{db_obj}->save; + + $c->track_progress(100); +} + +sub track_progress { + my ($self, $progress) = @_; + $self->{db_obj}->set_data(progress => $progress); + $self->{db_obj}->save; } sub cleanup { diff --git a/SL/Controller/CsvImport.pm b/SL/Controller/CsvImport.pm index a42654e7d..4db025e7d 100644 --- a/SL/Controller/CsvImport.pm +++ b/SL/Controller/CsvImport.pm @@ -24,7 +24,11 @@ use Rose::Object::MakeMethods::Generic ( scalar => [ qw(type profile file all_profiles all_charsets sep_char all_sep_chars quote_char all_quote_chars escape_char all_escape_chars all_buchungsgruppen all_units import_status errors headers raw_data_headers info_headers data num_imported num_importable displayable_columns file) ], - 'scalar --get_set_init' => [ qw(worker) ] + 'scalar --get_set_init' => [ qw(worker) ], + 'array' => [ + progress_tracker => { }, + add_progress_tracker => { interface => 'add', hash_key => 'progress_tracker' }, + ], ); __PACKAGE__->run_before('check_auth'); @@ -81,18 +85,14 @@ sub action_result { my $data = $self->{background_job}->data_as_hash; - my $profile = SL::DB::CsvImportProfile->new(type => $data->{type}); - my $profile_data = $data->{profile}; - for (keys %$profile_data) { - $profile->set($_ => $profile_data->{$_}); - } + my $profile = SL::DB::Manager::CsvImportProfile->find_by(id => $data->{profile_id}); $self->profile($profile); if ($data->{progress} < 100) { $self->render('csv_import/_deferred_results', { no_layout => 1 }); } else { - die 'what? done? panic, no idea what to do'; + $self->action_report(report_id => $data->{report_id}, no_layout => 1); } } @@ -221,19 +221,12 @@ sub test_and_import_deferred { sub test_and_import { my ($self, %params) = @_; - $self->profile_from_form; - - if ($::form->{file}) { - my $file = SL::SessionFile->new($self->csv_file_name, mode => '>'); - $file->fh->print($::form->{file}); - $file->fh->close; - } - - my $file = SL::SessionFile->new($self->csv_file_name, mode => '<', encoding => $self->profile->get('charset')); - if (!$file->fh) { - flash('error', $::locale->text('No file has been uploaded yet.')); - return $self->action_new; - } + my $file = SL::SessionFile->new( + $self->csv_file_name, + mode => '<', + encoding => $self->profile->get('charset'), + session_id => $params{session_id} + ); $self->file($file); @@ -244,12 +237,10 @@ sub test_and_import { $self->num_imported(0); $worker->save_objects if !$params{test}; - $self->save_report; - $self->num_importable(scalar grep { !$_ } map { scalar @{ $_->{errors} } } @{ $self->data || [] }); $self->import_status($params{test} ? 'tested' : 'imported'); - flash('info', $::locale->text('Objects have been imported.')) if !$params{test}; +# flash('info', $::locale->text('Objects have been imported.')) if !$params{test}; } sub load_default_profile { @@ -401,5 +392,13 @@ sub setup_help { $self->worker->setup_displayable_columns; } +sub track_progress { + my ($self, $progress) = @_; + + for my $tracker ($self->progress_tracker) { + $tracker->track_progress($progress); + } +} + 1; diff --git a/SL/Controller/CsvImport/Base.pm b/SL/Controller/CsvImport/Base.pm index 656495e37..67ca19185 100644 --- a/SL/Controller/CsvImport/Base.pm +++ b/SL/Controller/CsvImport/Base.pm @@ -22,6 +22,8 @@ use Rose::Object::MakeMethods::Generic sub run { my ($self) = @_; + $self->controller->track_progress(0); + my $profile = $self->profile; $self->csv(SL::Helper::Csv->new(file => $self->file->file_name, encoding => $self->controller->profile->get('charset'), @@ -33,11 +35,15 @@ sub run { map { ( $_ => $self->controller->profile->get($_) ) } qw(sep_char escape_char quote_char), )); + $self->controller->track_progress(1); + my $old_numberformat = $::myconfig{numberformat}; $::myconfig{numberformat} = $self->controller->profile->get('numberformat'); $self->csv->parse; + $self->controller->track_progress(3); + $self->controller->errors([ $self->csv->errors ]) if $self->csv->errors; return if ( !$self->csv->header || $self->csv->errors ); @@ -49,10 +55,22 @@ sub run { $self->controller->raw_data_headers({ used => { }, headers => [ ] }); $self->controller->info_headers({ used => { }, headers => [ ] }); + $::lxdebug->dump(0, "self", $self->controller->info_headers); + $::lxdebug->dump(0, "self", $self->controller->headers); + $::lxdebug->dump(0, "self", $self->controller->raw_data_headers); + my @objects = $self->csv->get_objects; + + $self->controller->track_progress(4); + my @raw_data = @{ $self->csv->get_data }; + + $self->controller->track_progress(4.5); + $self->controller->data([ pairwise { { object => $a, raw_data => $b, errors => [], information => [], info_data => {} } } @objects, @raw_data ]); + $self->controller->track_progress(5); + $self->check_objects; if ( $self->controller->profile->get('duplicates', 'no_check') ne 'no_check' ) { $self->check_std_duplicates(); @@ -60,6 +78,8 @@ sub run { } $self->fix_field_lengths; + $self->controller->track_progress(99); + $::myconfig{numberformat} = $old_numberformat; } @@ -364,7 +384,14 @@ sub save_objects { my $data = $params{data} || $self->controller->data; - foreach my $entry (@{ $data }) { + return unless $data->[0]; + return unless $data->[0]{object}; + + my $dbh = $data->[0]{object}->db; + + $dbh->begin_work; + foreach my $entry_index (0 .. $#$data) { + my $entry = $data->[$entry_index]; next if @{ $entry->{errors} }; my $object = $entry->{object_to_save} || $entry->{object}; @@ -374,7 +401,14 @@ sub save_objects { } else { $self->controller->num_imported($self->controller->num_imported + 1); } + } continue { + if ($entry_index % 100 == 0) { + $dbh->commit; + $self->controller->track_progress(45 + $entry_index/scalar(@$data) * 50); # scale from 45..95%; + $dbh->begin_work; + } } + $dbh->commit; } sub field_lengths { diff --git a/SL/Controller/CsvImport/Part.pm b/SL/Controller/CsvImport/Part.pm index 1538f2d9c..36dccc62a 100644 --- a/SL/Controller/CsvImport/Part.pm +++ b/SL/Controller/CsvImport/Part.pm @@ -105,7 +105,11 @@ sub check_objects { $self->makemodel_columns({}); + my $i; + my $num_data = scalar @{ $self->controller->data }; foreach my $entry (@{ $self->controller->data }) { + $self->controller->track_progress(8 + ($i/$num_data * 40)) if $i % 100 == 0; # scale from 5..45% + $self->check_buchungsgruppe($entry); $self->check_type($entry); $self->check_unit($entry); @@ -120,6 +124,8 @@ sub check_objects { $self->handle_cvars($entry); $self->handle_makemodel($entry); $self->set_various_fields($entry); + } continue { + $i++; } $self->add_columns(qw(type)) if $self->settings->{parts_type} eq 'mixed'; @@ -496,4 +502,4 @@ sub setup_displayable_columns { } } -1; \ No newline at end of file +1; -- 2.20.1