From: Sven Schöling <s.schoeling@linet-services.de>
Date: Thu, 15 Nov 2012 14:16:26 +0000 (+0100)
Subject: Weitere Verbesserungen am asynchronen Import.
X-Git-Tag: release-3.1.0beta1~733
X-Git-Url: http://wagnertech.de/git?a=commitdiff_plain;h=d74921657d967900bded4aaf4805647e9cf83562;p=kivitendo-erp.git

Weitere Verbesserungen am asynchronen Import.

- tracking
- profile/session handling
---

diff --git a/SL/BackgroundJob/CsvImport.pm b/SL/BackgroundJob/CsvImport.pm
index e34edc315..60147f8e2 100644
--- a/SL/BackgroundJob/CsvImport.pm
+++ b/SL/BackgroundJob/CsvImport.pm
@@ -5,6 +5,7 @@ use strict;
 use parent qw(SL::BackgroundJob::Base);
 
 use YAML ();
+use SL::Controller::CsvImport;
 use SL::DB::CsvImportProfile;
 use SL::SessionFile::Random;
 
@@ -15,12 +16,13 @@ sub create_job {
   $package          =~ s/SL::BackgroundJob:://;
 
   my $profile = delete $params{profile} || SL::DB::CsvImportProfile->new;
-  my $result  = delete $params{result}  || SL::SessionFile::Random->new;
+  my $new_profile = $profile->clone_and_reset_deep;
+  $new_profile->save;
 
   my %data = (
-    profile => { $profile->flatten },
-    result  => $result->file_name,
     %params,
+    profile_id => $new_profile->id,
+    session_id => $::auth->get_session_id,
   );
 
   my $job = SL::DB::BackgroundJob->new(
@@ -34,14 +36,11 @@ sub create_job {
 }
 
 sub profile {
-  my ($self, $db_obj) = @_;
+  my ($self) = @_;
 
   if (!$self->{profile}) {
-    $self->{profile} = SL::DB::CsvImportProfile->new;
-    my $data = YAML::Load($db_obj->data);
-    for (keys %$data) {
-      $self->{profile}->set($_ => $data->{$_});
-    }
+    my $data = YAML::Load($self->{db_obj}->data);
+    $self->{profile} = SL::DB::Manager::CsvImportProfile->find_by(id => $data->{profile_id});
   }
 
   return $self->{profile};
@@ -60,10 +59,25 @@ sub do_import {
   my ($self) = @_;
 
   my $c = SL::Controller::CsvImport->new;
-  $c->profile($self->{profile});
-  $c->test_and_import(test => $self->);
 
+  $c->profile($self->profile);
+  $c->type($self->{db_obj}->data_as_hash->{type});
+  $c->add_progress_tracker($self);
+
+  $c->test_and_import(test => 1, session_id => $self->{db_obj}->data_as_hash->{session_id});
+
+  my $report_id = $c->save_report;
+  $self->{db_obj}->set_data(report_id => $report_id);
+  $self->{db_obj}->save;
+
+  $c->track_progress(100);
+}
+
+sub track_progress {
+  my ($self, $progress) = @_;
 
+  $self->{db_obj}->set_data(progress => $progress);
+  $self->{db_obj}->save;
 }
 
 sub cleanup {
diff --git a/SL/Controller/CsvImport.pm b/SL/Controller/CsvImport.pm
index a42654e7d..4db025e7d 100644
--- a/SL/Controller/CsvImport.pm
+++ b/SL/Controller/CsvImport.pm
@@ -24,7 +24,11 @@ use Rose::Object::MakeMethods::Generic
 (
  scalar                  => [ qw(type profile file all_profiles all_charsets sep_char all_sep_chars quote_char all_quote_chars escape_char all_escape_chars all_buchungsgruppen all_units
                                  import_status errors headers raw_data_headers info_headers data num_imported num_importable displayable_columns file) ],
- 'scalar --get_set_init' => [ qw(worker) ]
+ 'scalar --get_set_init' => [ qw(worker) ],
+ 'array'                 => [
+   progress_tracker     => { },
+   add_progress_tracker => {  interface => 'add', hash_key => 'progress_tracker' },
+ ],
 );
 
 __PACKAGE__->run_before('check_auth');
@@ -81,18 +85,14 @@ sub action_result {
 
   my $data = $self->{background_job}->data_as_hash;
 
-  my $profile = SL::DB::CsvImportProfile->new(type => $data->{type});
-  my $profile_data = $data->{profile};
-  for (keys %$profile_data) {
-    $profile->set($_ => $profile_data->{$_});
-  }
+  my $profile = SL::DB::Manager::CsvImportProfile->find_by(id => $data->{profile_id});
 
   $self->profile($profile);
 
   if ($data->{progress} < 100) {
     $self->render('csv_import/_deferred_results', { no_layout => 1 });
   } else {
-   die 'what? done? panic, no idea what to do';
+    $self->action_report(report_id => $data->{report_id}, no_layout => 1);
   }
 }
 
@@ -221,19 +221,12 @@ sub test_and_import_deferred {
 sub test_and_import {
   my ($self, %params) = @_;
 
-  $self->profile_from_form;
-
-  if ($::form->{file}) {
-    my $file = SL::SessionFile->new($self->csv_file_name, mode => '>');
-    $file->fh->print($::form->{file});
-    $file->fh->close;
-  }
-
-  my $file = SL::SessionFile->new($self->csv_file_name, mode => '<', encoding => $self->profile->get('charset'));
-  if (!$file->fh) {
-    flash('error', $::locale->text('No file has been uploaded yet.'));
-    return $self->action_new;
-  }
+  my $file = SL::SessionFile->new(
+    $self->csv_file_name,
+    mode       => '<',
+    encoding   => $self->profile->get('charset'),
+    session_id => $params{session_id}
+  );
 
   $self->file($file);
 
@@ -244,12 +237,10 @@ sub test_and_import {
   $self->num_imported(0);
   $worker->save_objects if !$params{test};
 
-  $self->save_report;
-
   $self->num_importable(scalar grep { !$_ } map { scalar @{ $_->{errors} } } @{ $self->data || [] });
   $self->import_status($params{test} ? 'tested' : 'imported');
 
-  flash('info', $::locale->text('Objects have been imported.')) if !$params{test};
+#  flash('info', $::locale->text('Objects have been imported.')) if !$params{test};
 }
 
 sub load_default_profile {
@@ -401,5 +392,13 @@ sub setup_help {
   $self->worker->setup_displayable_columns;
 }
 
+sub track_progress {
+  my ($self, $progress) = @_;
+
+  for my $tracker ($self->progress_tracker) {
+    $tracker->track_progress($progress);
+  }
+}
+
 
 1;
diff --git a/SL/Controller/CsvImport/Base.pm b/SL/Controller/CsvImport/Base.pm
index 656495e37..67ca19185 100644
--- a/SL/Controller/CsvImport/Base.pm
+++ b/SL/Controller/CsvImport/Base.pm
@@ -22,6 +22,8 @@ use Rose::Object::MakeMethods::Generic
 sub run {
   my ($self) = @_;
 
+  $self->controller->track_progress(0);
+
   my $profile = $self->profile;
   $self->csv(SL::Helper::Csv->new(file                   => $self->file->file_name,
                                   encoding               => $self->controller->profile->get('charset'),
@@ -33,11 +35,15 @@ sub run {
                                   map { ( $_ => $self->controller->profile->get($_) ) } qw(sep_char escape_char quote_char),
                                  ));
 
+  $self->controller->track_progress(1);
+
   my $old_numberformat      = $::myconfig{numberformat};
   $::myconfig{numberformat} = $self->controller->profile->get('numberformat');
 
   $self->csv->parse;
 
+  $self->controller->track_progress(3);
+
   $self->controller->errors([ $self->csv->errors ]) if $self->csv->errors;
 
   return if ( !$self->csv->header || $self->csv->errors );
@@ -49,10 +55,22 @@ sub run {
   $self->controller->raw_data_headers({ used => { }, headers => [ ] });
   $self->controller->info_headers({ used => { }, headers => [ ] });
 
+  $::lxdebug->dump(0,  "self", $self->controller->info_headers);
+  $::lxdebug->dump(0,  "self", $self->controller->headers);
+  $::lxdebug->dump(0,  "self", $self->controller->raw_data_headers);
+
   my @objects  = $self->csv->get_objects;
+
+  $self->controller->track_progress(4);
+
   my @raw_data = @{ $self->csv->get_data };
+
+  $self->controller->track_progress(4.5);
+
   $self->controller->data([ pairwise { { object => $a, raw_data => $b, errors => [], information => [], info_data => {} } } @objects, @raw_data ]);
 
+  $self->controller->track_progress(5);
+
   $self->check_objects;
   if ( $self->controller->profile->get('duplicates', 'no_check') ne 'no_check' ) {
     $self->check_std_duplicates();
@@ -60,6 +78,8 @@ sub run {
   }
   $self->fix_field_lengths;
 
+  $self->controller->track_progress(99);
+
   $::myconfig{numberformat} = $old_numberformat;
 }
 
@@ -364,7 +384,14 @@ sub save_objects {
 
   my $data = $params{data} || $self->controller->data;
 
-  foreach my $entry (@{ $data }) {
+  return unless $data->[0];
+  return unless $data->[0]{object};
+
+  my $dbh = $data->[0]{object}->db;
+
+  $dbh->begin_work;
+  foreach my $entry_index (0 .. $#$data) {
+    my $entry = $data->[$entry_index];
     next if @{ $entry->{errors} };
 
     my $object = $entry->{object_to_save} || $entry->{object};
@@ -374,7 +401,14 @@ sub save_objects {
     } else {
       $self->controller->num_imported($self->controller->num_imported + 1);
     }
+  } continue {
+    if ($entry_index % 100 == 0) {
+      $dbh->commit;
+      $self->controller->track_progress(45 + $entry_index/scalar(@$data) * 50); # scale from 45..95%;
+      $dbh->begin_work;
+    }
   }
+  $dbh->commit;
 }
 
 sub field_lengths {
diff --git a/SL/Controller/CsvImport/Part.pm b/SL/Controller/CsvImport/Part.pm
index 1538f2d9c..36dccc62a 100644
--- a/SL/Controller/CsvImport/Part.pm
+++ b/SL/Controller/CsvImport/Part.pm
@@ -105,7 +105,11 @@ sub check_objects {
 
   $self->makemodel_columns({});
 
+  my $i;
+  my $num_data = scalar @{ $self->controller->data };
   foreach my $entry (@{ $self->controller->data }) {
+    $self->controller->track_progress(8 + ($i/$num_data * 40)) if $i % 100 == 0; # scale from 5..45%
+
     $self->check_buchungsgruppe($entry);
     $self->check_type($entry);
     $self->check_unit($entry);
@@ -120,6 +124,8 @@ sub check_objects {
     $self->handle_cvars($entry);
     $self->handle_makemodel($entry);
     $self->set_various_fields($entry);
+  } continue {
+    $i++;
   }
 
   $self->add_columns(qw(type)) if $self->settings->{parts_type} eq 'mixed';
@@ -496,4 +502,4 @@ sub setup_displayable_columns {
   }
 }
 
-1;
\ No newline at end of file
+1;