1 package SL::Helper::ShippedQty;
4 use parent qw(Rose::Object);
7 use Scalar::Util qw(blessed);
8 use List::Util qw(min);
9 use List::MoreUtils qw(any all uniq);
10 use List::UtilsBy qw(partition_by);
12 use SL::DBUtils qw(selectall_hashref_query selectall_as_map);
13 use SL::Locale::String qw(t8);
15 use Rose::Object::MakeMethods::Generic (
16 'scalar' => [ qw(objects objects_or_ids shipped_qty keep_matches) ],
17 'scalar --get_set_init' => [ qw(oe_ids dbh require_stock_out fill_up item_identity_fields oi2oe oi_qty delivered matches
18 services_deliverable) ],
21 my $no_stock_item_links_query = <<'';
22 SELECT oi.trans_id, oi.id AS oi_id, oi.qty AS oi_qty, oi.unit AS oi_unit, doi.id AS doi_id, doi.qty AS doi_qty, doi.unit AS doi_unit
24 INNER JOIN orderitems oi ON oi.id = rl.from_id AND rl.from_table = 'orderitems'
25 INNER JOIN delivery_order_items doi ON doi.id = rl.to_id AND rl.to_table = 'delivery_order_items'
26 WHERE oi.trans_id IN (%s)
27 ORDER BY oi.trans_id, oi.position
29 # oi not item linked. takes about 250ms for 100k hits
30 # obsolete since 3.5.6
31 my $fill_up_oi_query = <<'';
32 SELECT oi.id, oi.trans_id, oi.position, oi.parts_id, oi.description, oi.reqdate, oi.serialnumber, oi.qty, oi.unit
34 WHERE oi.trans_id IN (%s)
35 ORDER BY oi.trans_id, oi.position
37 # doi linked by record, but not by items; 250ms for 100k hits
38 # obsolete since 3.5.6
39 my $no_stock_fill_up_doi_query = <<'';
40 SELECT doi.id, doi.delivery_order_id, doi.position, doi.parts_id, doi.description, doi.reqdate, doi.serialnumber, doi.qty, doi.unit
41 FROM delivery_order_items doi
42 WHERE doi.delivery_order_id IN (
47 AND to_table = 'delivery_orders'
48 AND to_id = doi.delivery_order_id)
52 WHERE from_table = 'orderitems'
53 AND to_table = 'delivery_order_items'
56 my $stock_item_links_query = <<'';
57 SELECT oi.trans_id, oi.id AS oi_id, oi.qty AS oi_qty, oi.unit AS oi_unit, doi.id AS doi_id,
58 (CASE WHEN doe.customer_id > 0 THEN -1 ELSE 1 END) * i.qty AS doi_qty, p.unit AS doi_unit
60 INNER JOIN orderitems oi ON oi.id = rl.from_id AND rl.from_table = 'orderitems'
61 INNER JOIN delivery_order_items doi ON doi.id = rl.to_id AND rl.to_table = 'delivery_order_items'
62 INNER JOIN delivery_orders doe ON doe.id = doi.delivery_order_id
63 INNER JOIN delivery_order_items_stock dois ON dois.delivery_order_item_id = doi.id
64 INNER JOIN inventory i ON dois.id = i.delivery_order_items_stock_id
65 INNER JOIN parts p ON p.id = doi.parts_id
66 WHERE oi.trans_id IN (%s)
67 ORDER BY oi.trans_id, oi.position
69 my $stock_fill_up_doi_query = <<'';
70 SELECT doi.id, doi.delivery_order_id, doi.position, doi.parts_id, doi.description, doi.reqdate, doi.serialnumber,
71 (CASE WHEN doe.customer_id > 0 THEN -1 ELSE 1 END) * i.qty, p.unit
72 FROM delivery_order_items doi
73 INNER JOIN parts p ON p.id = doi.parts_id
74 INNER JOIN delivery_order_items_stock dois ON dois.delivery_order_item_id = doi.id
75 INNER JOIN delivery_orders doe ON doe.id = doi.delivery_order_id
76 INNER JOIN inventory i ON dois.id = i.delivery_order_items_stock_id
77 WHERE doi.delivery_order_id IN (
82 AND to_table = 'delivery_orders'
83 AND to_id = doi.delivery_order_id)
87 WHERE from_table = 'orderitems'
88 AND to_table = 'delivery_order_items'
91 my $oe_do_record_links = <<'';
96 AND to_table = 'delivery_orders'
98 my @known_item_identity_fields = qw(parts_id description reqdate serialnumber);
99 my %item_identity_fields = (
100 parts_id => t8('Part'),
101 description => t8('Description'),
102 reqdate => t8('Reqdate'),
103 serialnumber => t8('Serial Number'),
107 my ($self, $data) = @_;
109 croak 'Need exactly one argument, either id, object or arrayref of ids or objects.' unless 2 == @_;
111 $self->normalize_input($data);
113 return $self unless @{ $self->oe_ids };
115 $self->calculate_item_links;
116 $self->calculate_fill_up if $self->fill_up;
121 sub calculate_item_links {
124 my @oe_ids = @{ $self->oe_ids };
126 my $item_links_query = $self->require_stock_out ? $stock_item_links_query : $no_stock_item_links_query;
128 my $query = sprintf $item_links_query, join (', ', ('?')x @oe_ids);
130 my $data = selectall_hashref_query($::form, $self->dbh, $query, @oe_ids);
133 my $qty = $_->{doi_qty} * AM->convert_unit($_->{doi_unit} => $_->{oi_unit});
134 $self->shipped_qty->{$_->{oi_id}} //= 0;
135 $self->shipped_qty->{$_->{oi_id}} += $qty;
136 $self->oi2oe->{$_->{oi_id}} = $_->{trans_id};
137 $self->oi_qty->{$_->{oi_id}} = $_->{oi_qty};
139 push @{ $self->matches }, [ $_->{oi_id}, $_->{doi_id}, $qty, 1 ] if $self->keep_matches;
146 grep { $seen{$_}++ } @$a1, @$a2;
149 sub calculate_fill_up {
152 my @oe_ids = @{ $self->oe_ids };
154 my $fill_up_doi_query = $self->require_stock_out ? $stock_fill_up_doi_query : $no_stock_fill_up_doi_query;
156 my $oi_query = sprintf $fill_up_oi_query, join (', ', ('?')x@oe_ids);
157 my $doi_query = sprintf $fill_up_doi_query, join (', ', ('?')x@oe_ids);
158 my $rl_query = sprintf $oe_do_record_links, join (', ', ('?')x@oe_ids);
160 my $oi = selectall_hashref_query($::form, $self->dbh, $oi_query, @oe_ids);
164 my $doi = selectall_hashref_query($::form, $self->dbh, $doi_query, @oe_ids);
165 my $rl = selectall_hashref_query($::form, $self->dbh, $rl_query, @oe_ids);
167 my %oi_by_identity = partition_by { $self->item_identity($_) } @$oi;
168 my %doi_by_id = partition_by { $_->{delivery_order_id} } @$doi;
170 push @{ $doi_by_trans_id{$_->{from_id}} //= [] }, @{ $doi_by_id{$_->{to_id}} }
171 for grep { exists $doi_by_id{$_->{to_id}} } @$rl;
173 my %doi_by_identity = partition_by { $self->item_identity($_) } @$doi;
175 for my $match (sort keys %oi_by_identity) {
176 next unless exists $doi_by_identity{$match};
178 my %oi_by_oe = partition_by { $_->{trans_id} } @{ $oi_by_identity{$match} };
179 for my $trans_id (sort { $a <=> $b } keys %oi_by_oe) {
180 next unless my @sorted_doi = _intersect($doi_by_identity{$match}, $doi_by_trans_id{$trans_id});
182 # sorting should be quite fast here, because there are usually only a handful of matches
183 next unless my @sorted_oi = sort { $a->{position} <=> $b->{position} } @{ $oi_by_oe{$trans_id} };
185 # parallel walk through sorted oi/doi entries
186 my $oi_i = my $doi_i = 0;
187 my ($oi, $doi) = ($sorted_oi[$oi_i], $sorted_doi[$doi_i]);
188 while ($oi_i < @sorted_oi && $doi_i < @sorted_doi) {
189 $oi = $sorted_oi[++$oi_i], next if $oi->{qty} <= $self->shipped_qty->{$oi->{id}};
190 $doi = $sorted_doi[++$doi_i], next if 0 == $doi->{qty};
192 my $factor = AM->convert_unit($doi->{unit} => $oi->{unit});
193 my $min_qty = min($oi->{qty} - $self->shipped_qty->{$oi->{id}}, $doi->{qty} * $factor);
195 # min_qty should never be 0 now. the first part triggers the first next,
196 # the second triggers the second next and factor must not be 0
197 # but it would lead to an infinite loop, so catch that.
198 die 'panic! invalid shipping quantity' unless $min_qty;
200 $self->shipped_qty->{$oi->{id}} += $min_qty;
201 $doi->{qty} -= $min_qty / $factor; # TODO: find a way to avoid float rounding
202 push @{ $self->matches }, [ $oi->{id}, $doi->{id}, $min_qty, 0 ] if $self->keep_matches;
207 $self->oi2oe->{$_->{id}} = $_->{trans_id} for @$oi;
208 $self->oi_qty->{$_->{id}} = $_->{qty} for @$oi;
212 my ($self, $objects) = @_;
214 croak 'expecting array of objects' unless 'ARRAY' eq ref $objects;
216 my $shipped_qty = $self->shipped_qty;
218 for my $obj (@$objects) {
219 if ('SL::DB::OrderItem' eq ref $obj) {
220 $obj->{shipped_qty} = $shipped_qty->{$obj->id} //= 0;
221 $obj->{delivered} = $shipped_qty->{$obj->id} == $obj->qty;
222 } elsif ('SL::DB::Order' eq ref $obj) {
223 # load all orderitems unless not already loaded
224 $obj->orderitems unless (defined $obj->{orderitems});
225 $self->write_to($obj->{orderitems});
226 if ($self->services_deliverable) {
227 $obj->{delivered} = all { $_->{delivered} } grep { !$_->{optional} } @{ $obj->{orderitems} };
229 $obj->{delivered} = all { $_->{delivered} } grep { !$_->{optional} && !$_->part->is_service } @{ $obj->{orderitems} };
232 die "unknown reference '@{[ ref $obj ]}' for @{[ __PACKAGE__ ]}::write_to";
238 sub write_to_objects {
241 return unless @{ $self->oe_ids };
243 croak 'Can only use write_to_objects, when calculate was called with objects. Use write_to instead.' unless $self->objects_or_ids;
245 $self->write_to($self->objects);
249 my ($self, $row) = @_;
251 join $;, map $row->{$_}, @{ $self->item_identity_fields };
254 sub normalize_input {
255 my ($self, $data) = @_;
257 $data = [$data] if 'ARRAY' ne ref $data;
259 $self->objects_or_ids(!!blessed($data->[0]));
261 if ($self->objects_or_ids) {
262 croak 'unblessed object in data while expecting object' if any { !blessed($_) } @$data;
263 $self->objects($data);
265 croak 'object or reference in data while expecting ids' if any { ref($_) } @$data;
266 croak 'ids need to be numbers' if any { ! ($_ * 1) } @$data;
267 $self->oe_ids($data);
270 $self->shipped_qty({});
273 sub available_item_identity_fields {
274 map { [ $_ => $item_identity_fields{$_} ] } @known_item_identity_fields;
280 croak 'oe_ids not initialized in id mode' if !$self->objects_or_ids;
281 croak 'objects not initialized before accessing ids' if $self->objects_or_ids && !defined $self->objects;
282 croak 'objects need to be Order or OrderItem' if any { ref($_) !~ /^SL::DB::Order(?:Item)?$/ } @{ $self->objects };
284 [ uniq map { ref($_) =~ /Item/ ? $_->trans_id : $_->id } @{ $self->objects } ]
287 sub init_dbh { SL::DB->client->dbh }
289 sub init_oi2oe { {} }
290 sub init_oi_qty { {} }
291 sub init_matches { [] }
295 for (keys %{ $self->oi_qty }) {
296 my $oe_id = $self->oi2oe->{$_};
298 $d->{$oe_id} &&= $self->shipped_qty->{$_} == $self->oi_qty->{$_};
303 sub init_require_stock_out { $::instance_conf->get_shipped_qty_require_stock_out }
304 sub init_item_identity_fields { [ grep $item_identity_fields{$_}, @{ $::instance_conf->get_shipped_qty_item_identity_fields } ] }
305 sub init_fill_up { $::instance_conf->get_shipped_qty_fill_up }
307 sub init_services_deliverable {
309 if ($::form->{type} =~ m/^sales_/) {
310 $::instance_conf->get_sales_delivery_order_check_service;
311 } elsif ($::form->{type} =~ m/^purchase_/) {
312 $::instance_conf->get_purchase_delivery_order_check_service;
314 croak "wrong call, no customer or vendor object referenced";
326 SL::Helper::ShippedQty - Algorithmic module for calculating shipped qty
330 use SL::Helper::ShippedQty;
332 my $helper = SL::Helper::ShippedQty->new(
334 require_stock_out => 0,
335 item_identity_fields => [ qw(parts_id description reqdate serialnumber) ],
338 $helper->calculate($order_object);
339 $helper->calculate(\@order_objects);
340 $helper->calculate($orderitem_object);
341 $helper->calculate(\@orderitem_objects);
342 $helper->calculate($oe_id);
343 $helper->calculate(\@oe_ids);
345 # if these are items set delivered and shipped_qty
346 # if these are orders, iterate through their items and set delivered on order
347 $helper->write_to($objects);
349 # if calculate was called with objects, you can use this shortcut:
350 $helper->write_to_objects;
352 # shipped_qtys by oi_id
353 my $shipped_qty = $helper->shipped_qty->{$oi->id};
356 my $delivered = $helper->delievered->{$oi->id};
358 # calculate and write_to can be chained:
359 my $helper = SL::Helper::ShippedQty->new->calculate($orders)->write_to_objects;
363 This module encapsulates the algorithm needed to compute the shipped qty for
364 orderitems (hopefully) correctly and efficiently for several use cases.
366 While this is used in object accessors, it can not be fast when called in a
367 loop over and over, so take advantage of batch processing when possible.
369 =head1 MOTIVATION AND PROBLEMS
371 The concept of shipped qty is sadly not as straight forward as it sounds at
372 first glance. Any correct implementation must in some way deal with the
379 When is an order shipped? For users that use the inventory it
380 will mean when a delivery order is stocked out. For those not using the
381 inventory it will mean when the delivery order is saved.
385 How to find the correct matching elements. After the changes
386 to record item links it's natural to assume that each position is linked, but
387 for various reasons this might not be the case. Positions that are not linked
388 in the database need to be matched by marching.
392 Double links need to be accounted for (these can stem from buggy code).
396 orderitems and oe entries may link to many of their counterparts in
397 delivery_orders. delivery_orders my be created from multiple orders. The
398 only constant is that a single entry in delivery_order_items has at most one
399 link from an orderitem.
403 For the fill up case the identity of positions is not clear. The naive approach
404 is just the same part, but description, charge number, reqdate and qty can all
405 be part of the identity of a position for finding shipped matches.
409 Certain delivery orders might not be eligible for qty calculations if delivery
410 orders are used for other purposes.
414 Units need to be handled correctly
418 Negative positions must be taken into account. A negative delivery order is
419 assumed to be a RMA of sorts, but a negative order is not as straight forward.
423 Must be able to work with plain ids and Rose objects, and absolutely must
424 include a bulk mode to speed up multiple objects.
435 Creates a new helper object. PARAMS may include:
439 =item * C<require_stock_out>
441 Boolean. If set, delivery orders must be stocked out to be considered
442 delivered. The default is a client setting.
446 Boolean. If set, unlinked delivery order items will be used to fill up
447 undelivered order items. Not needed in newer installations. The default is a
450 =item * C<item_identity_fields ARRAY>
452 If set, the fields are used to compute the identity of matching positions. The
453 default is a client setting. Possible values include:
459 =item * C<description>
463 =item * C<serialnumber>
467 =item * C<keep_matches>
469 Boolean. If set to true the internal matchings of OrderItems and
470 DeliveryOrderItems will be kept for later postprocessing, in case you need more
471 than this modules provides.
473 See C<matches> for the returned format.
477 =item C<calculate OBJECTS>
479 =item C<calculate IDS>
481 Do the main work. There must be a single argument: Either an id or an
482 C<SL::DB::Order> object, or an arrayref of one of these types.
484 Mixing ids and objects will generate an exception.
486 No return value. All internal errors will throw an exception.
488 =item C<write_to OBJECTS>
490 =item C<write_to_objects>
492 Save the C<shipped_qty> and C<delivered> state to the given objects. If
493 L</calculate> was called with objects, then C<write_to_objects> will use these.
495 C<shipped_qty> and C<delivered> will be directly infused into the objects
496 without calling the accessor for delivered. If you want to save afterwards,
497 you'll have to do that yourself.
499 C<shipped_qty> is guaranteed to be coerced to a number. If no delivery_order
500 was found it will be set to zero.
502 C<delivered> is guaranteed only to be the correct boolean value, but not
505 Note: C<write_to> will avoid loading unnecessary objects. This means if it is
506 called with an Order object that has not loaded its orderitems yet, only
507 C<delivered> will be set in the Order object. A subsequent C<<
508 $order->orderitems->[0]->{delivered} >> will return C<undef>, and C<<
509 $order->orderitems->[0]->shipped_qty >> will invoke another implicit
514 Valid after L</calculate>. Returns a hasref with shipped qtys by orderitems id.
516 Unlike the result of C</write_to>, entries in C<shipped_qty> may be C<undef> if
517 linked elements were found.
521 Valid after L</calculate>. Returns a hashref with a delivered flag by order id.
525 Valid after L</calculate> with C<with_matches> set. Returns an arrayref of
526 individual matches. Each match is an arrayref with these fields:
532 The id of the OrderItem.
536 The id of the DeliveryOrderItem.
540 The qty that was matched between the two converted to the unit of the OrderItem.
544 A boolean flag indicating if this match was found with record_item links. If
545 false, the match was made in the fill up stage.
551 =head1 REPLACED FUNCTIONALITY
553 =head2 delivered mode
555 Originally used in mark_orders_if_delivered. Searches for orders associated
556 with a delivery order and evaluates whether those are delivered or not. No
557 detailed information is needed.
559 This is to be integrated into fast delivered check on the orders. The calling
560 convention for the delivery_order is not part of the scope of this module.
564 Originally used for printing delivery orders. Resolves for each position for
565 how much was originally ordered, and how much remains undelivered.
567 This one is likely to be dropped. The information only makes sense without
568 combined merge/split deliveries and is very fragile with unaccounted delivery
573 Same from the order perspective. Used for transitions to delivery orders, where
574 delivered qtys should be removed from positions. Also used each time a record
575 is rendered to show the shipped qtys. Also used to find orders that are not
578 Acceptable shortcuts would be the concepts fully shipped (for the order) and
579 providing already loaded objects.
581 =head2 Replaces the following functions
583 C<DO::get_shipped_qty>
585 C<SL::Controller::DeliveryPlan::calc_qtys>
587 C<SL::DB::OrderItem::shipped_qty>
589 C<SL::DB::OrderItem::delivered_qty>
593 this is the old get_shipped_qty algorithm by Martin for reference
595 in: oe_id, do_id, doctype, delivered flag
597 not needed with better signatures
599 load oe->do links for this id,
600 set oe_ids from those
607 2 load all orderitems for these oe_ids
610 set undelivered := qty
613 create tuple: [ position => qty_ordered, qty_not_delivered, orderitem.id ]
615 1 load all oe->do links for these oe_ids
618 return all tuples so far
621 4 create dictionary for orderitems from [2] by id
623 3 load all delivery_order_items for do_ids from [1], with recorditem_links from orderitems
624 - optionally with doctype filter (identity filter)
626 # first pass for record_item_links
629 if link from orderitem exists and orderitem is in dictionary [4]
630 reduce qty_notdelivered in orderitem by doi.qty
631 keep link to do entry in orderitem
634 # second pass fill up
636 ignroe if from link exists or qty == 0
638 for orderitems from [2]:
639 next if notdelivered_qty == 0
640 if doi.parts_id == orderitem.parts_id:
641 if oi.notdelivered_qty < 0:
642 doi :+= -oi.notdelivered_qty,
643 oi.notdelivered_qty := 0
645 fi doi.qty < oi.notdelivered_qty:
647 oi.notdelivered_qty :-= doi.qty
649 doi.qty :-= oi.notdelivered_qty
650 oi.notdelivered_qty := 0
652 keep link to oi in doi
655 last wenn doi.qty <= 0
659 # post process for return
662 copy notdelivered from oe to ship{position}{notdelivered}
663 if !oe_id and do_id and delivered:
664 ship.{oi.trans_id}.delivered := oi.notdelivered_qty <= 0
665 if !oe_id and do_id and !delivered:
667 ignore if do.id != doi.delivery_order_id
668 if oi in doi verlinkt und position bekannt:
669 addiere oi.qty zu doi.ordered_qty
670 addiere oi.notdelievered_qty zu doi.notdelivered_qty
677 in: orders, parameters
679 normalize orders to ids
681 # handle record_item links
682 retrieve record_links entries with inner joins on orderitems, delivery_orderitems and stock/inventory if requested
683 for all record_links:
684 initialize shipped_qty for this doi to 0 if not yet seen
685 convert doi.qty to oi.unit
686 add normalized doi.qty to shipped_qty
690 abort if fill up is not requested
692 retrieve all orderitems matching the given order ids
693 retrieve all doi with a link to the given order ids but without item link (and optionally with stock/inventory)
694 retrieve all record_links between orders and delivery_orders (1)
696 abort when no dois were found
698 create a partition of the delivery order items by do_id (2)
699 create empty mapping for delivery order items by order_id (3)
700 for all record_links from [1]:
701 add all matching doi from (2) to (3)
704 create a partition of the orderitems by item identity (4)
705 create a partition of the delivery order items by item identity (5)
707 for each identity in (4):
708 skip if no matching entries in (5)
710 create partition of all orderitems for this identity by order id (6)
711 for each sorted order id in [6]:
712 look up matching delivery order items by identity from [5] (7)
713 look up matching delivery order items by order id from [3] (8)
714 create stable sorted intersection between [7] and [8] (9)
716 sort the orderitems from (6) by position (10)
718 parallel walk through [9] and [10]:
719 missing qty := oi.qty - shipped_qty[oi]
722 next orderitem if missing_qty <= 0
723 next delivery order item if doi.qty == 0
725 min_qty := minimum(missing_qty, [doi.qty converted to oi.unit]
727 # transfer min_qty from doi.qty to shipped[qty]:
728 shipped_qty[oi] += min_qty
729 doi.qty -= [min_qty converted to doi.unit]
734 =head1 COMPLEXITY OBSERVATIONS
736 Perl ops except for sort are expected to be constant (relative to the op overhead).
738 =head2 Record item links
740 The query itself has indices available for all joins and filters and should
741 scale with sublinear with the number of affected orderitems.
743 The rest of the code iterates through the result and calls C<AM::convert_unit>,
744 which caches internally and is asymptotically constant.
748 C<partition_by> and C<intersect> both scale linearly. The first two scale with
749 input size, but use existing indices. The delivery order items query scales
750 with the nested loop anti join of the "NOT EXISTS" subquery, which takes most
751 of the time. For large databases omitting the order id filter may be faster.
753 Three partitions after that scale linearly. Building the doi_by_oe_id
754 multimap is O(n²) worst case, but will be linear for most real life data.
756 Iterating through the values of the partitions scales with the number of
757 elements in the multimap, and does not add additional complexity.
759 The sort and parallel walk are O(nlogn) for the length of the subdivisions,
760 which again makes square worst case, but much less than that in the general
763 =head3 Space requirements
765 In the current form the results of the 4 queries get fetched, and 4 of them are
766 held in memory at the same time. Three persistent structures are held:
767 C<shipped_qty>, C<oi2oe>, and C<oi_qty> - all hashes with one entry for each
768 orderitem. C<delivered> is calculated on demand and is a hash with an entry for
769 each order id of input.
771 Temporary structures are partitions of the orderitems, of which again the fill
772 up multi map between order id and delivery order items is potentially the
773 largest with square requierment worst case.
778 * delivery order identity
780 * rewrite to avoid division
781 * rewrite to avoid selectall for really large queries (no problem for up to 100k)
782 * calling mode or return to flag delivery_orders as delivered?
783 * add localized field white list
784 * reduce worst case square space requirement to linear
788 None yet, but there are most likely a lot in code this funky.
792 Sven Schöling E<lt>s.schoeling@linet-services.deE<gt>