@@ -697,11 +697,41 @@ static bool throtl_slice_used(struct throtl_grp *tg, bool rw)
697697 return true;
698698}
699699
700+ static unsigned int calculate_io_allowed (u32 iops_limit ,
701+ unsigned long jiffy_elapsed )
702+ {
703+ unsigned int io_allowed ;
704+ u64 tmp ;
705+
706+ /*
707+ * jiffy_elapsed should not be a big value as minimum iops can be
708+ * 1 then at max jiffy elapsed should be equivalent of 1 second as we
709+ * will allow dispatch after 1 second and after that slice should
710+ * have been trimmed.
711+ */
712+
713+ tmp = (u64 )iops_limit * jiffy_elapsed ;
714+ do_div (tmp , HZ );
715+
716+ if (tmp > UINT_MAX )
717+ io_allowed = UINT_MAX ;
718+ else
719+ io_allowed = tmp ;
720+
721+ return io_allowed ;
722+ }
723+
724+ static u64 calculate_bytes_allowed (u64 bps_limit , unsigned long jiffy_elapsed )
725+ {
726+ return mul_u64_u64_div_u64 (bps_limit , (u64 )jiffy_elapsed , (u64 )HZ );
727+ }
728+
700729/* Trim the used slices and adjust slice start accordingly */
701730static inline void throtl_trim_slice (struct throtl_grp * tg , bool rw )
702731{
703- unsigned long nr_slices , time_elapsed , io_trim ;
704- u64 bytes_trim , tmp ;
732+ unsigned long time_elapsed ;
733+ long long bytes_trim ;
734+ int io_trim ;
705735
706736 BUG_ON (time_before (tg -> slice_end [rw ], tg -> slice_start [rw ]));
707737
@@ -723,67 +753,38 @@ static inline void throtl_trim_slice(struct throtl_grp *tg, bool rw)
723753
724754 throtl_set_slice_end (tg , rw , jiffies + tg -> td -> throtl_slice );
725755
726- time_elapsed = jiffies - tg -> slice_start [rw ];
727-
728- nr_slices = time_elapsed / tg -> td -> throtl_slice ;
729-
730- if (!nr_slices )
756+ time_elapsed = rounddown (jiffies - tg -> slice_start [rw ],
757+ tg -> td -> throtl_slice );
758+ if (!time_elapsed )
731759 return ;
732- tmp = tg_bps_limit (tg , rw ) * tg -> td -> throtl_slice * nr_slices ;
733- do_div (tmp , HZ );
734- bytes_trim = tmp ;
735760
736- io_trim = (tg_iops_limit (tg , rw ) * tg -> td -> throtl_slice * nr_slices ) /
737- HZ ;
738-
739- if (!bytes_trim && !io_trim )
761+ bytes_trim = calculate_bytes_allowed (tg_bps_limit (tg , rw ),
762+ time_elapsed ) +
763+ tg -> carryover_bytes [rw ];
764+ io_trim = calculate_io_allowed (tg_iops_limit (tg , rw ), time_elapsed ) +
765+ tg -> carryover_ios [rw ];
766+ if (bytes_trim <= 0 && io_trim <= 0 )
740767 return ;
741768
742- if (tg -> bytes_disp [rw ] >= bytes_trim )
769+ tg -> carryover_bytes [rw ] = 0 ;
770+ if ((long long )tg -> bytes_disp [rw ] >= bytes_trim )
743771 tg -> bytes_disp [rw ] -= bytes_trim ;
744772 else
745773 tg -> bytes_disp [rw ] = 0 ;
746774
747- if (tg -> io_disp [rw ] >= io_trim )
775+ tg -> carryover_ios [rw ] = 0 ;
776+ if ((int )tg -> io_disp [rw ] >= io_trim )
748777 tg -> io_disp [rw ] -= io_trim ;
749778 else
750779 tg -> io_disp [rw ] = 0 ;
751780
752- tg -> slice_start [rw ] += nr_slices * tg -> td -> throtl_slice ;
781+ tg -> slice_start [rw ] += time_elapsed ;
753782
754783 throtl_log (& tg -> service_queue ,
755- "[%c] trim slice nr=%lu bytes=%llu io=%lu start=%lu end=%lu jiffies=%lu" ,
756- rw == READ ? 'R' : 'W' , nr_slices , bytes_trim , io_trim ,
757- tg -> slice_start [rw ], tg -> slice_end [rw ], jiffies );
758- }
759-
760- static unsigned int calculate_io_allowed (u32 iops_limit ,
761- unsigned long jiffy_elapsed )
762- {
763- unsigned int io_allowed ;
764- u64 tmp ;
765-
766- /*
767- * jiffy_elapsed should not be a big value as minimum iops can be
768- * 1 then at max jiffy elapsed should be equivalent of 1 second as we
769- * will allow dispatch after 1 second and after that slice should
770- * have been trimmed.
771- */
772-
773- tmp = (u64 )iops_limit * jiffy_elapsed ;
774- do_div (tmp , HZ );
775-
776- if (tmp > UINT_MAX )
777- io_allowed = UINT_MAX ;
778- else
779- io_allowed = tmp ;
780-
781- return io_allowed ;
782- }
783-
784- static u64 calculate_bytes_allowed (u64 bps_limit , unsigned long jiffy_elapsed )
785- {
786- return mul_u64_u64_div_u64 (bps_limit , (u64 )jiffy_elapsed , (u64 )HZ );
784+ "[%c] trim slice nr=%lu bytes=%lld io=%d start=%lu end=%lu jiffies=%lu" ,
785+ rw == READ ? 'R' : 'W' , time_elapsed / tg -> td -> throtl_slice ,
786+ bytes_trim , io_trim , tg -> slice_start [rw ], tg -> slice_end [rw ],
787+ jiffies );
787788}
788789
789790static void __tg_update_carryover (struct throtl_grp * tg , bool rw )
@@ -816,7 +817,7 @@ static void tg_update_carryover(struct throtl_grp *tg)
816817 __tg_update_carryover (tg , WRITE );
817818
818819 /* see comments in struct throtl_grp for meaning of these fields. */
819- throtl_log (& tg -> service_queue , "%s: %llu %llu %u %u \n" , __func__ ,
820+ throtl_log (& tg -> service_queue , "%s: %lld %lld %d %d \n" , __func__ ,
820821 tg -> carryover_bytes [READ ], tg -> carryover_bytes [WRITE ],
821822 tg -> carryover_ios [READ ], tg -> carryover_ios [WRITE ]);
822823}
@@ -825,7 +826,7 @@ static unsigned long tg_within_iops_limit(struct throtl_grp *tg, struct bio *bio
825826 u32 iops_limit )
826827{
827828 bool rw = bio_data_dir (bio );
828- unsigned int io_allowed ;
829+ int io_allowed ;
829830 unsigned long jiffy_elapsed , jiffy_wait , jiffy_elapsed_rnd ;
830831
831832 if (iops_limit == UINT_MAX ) {
@@ -838,9 +839,8 @@ static unsigned long tg_within_iops_limit(struct throtl_grp *tg, struct bio *bio
838839 jiffy_elapsed_rnd = roundup (jiffy_elapsed + 1 , tg -> td -> throtl_slice );
839840 io_allowed = calculate_io_allowed (iops_limit , jiffy_elapsed_rnd ) +
840841 tg -> carryover_ios [rw ];
841- if (tg -> io_disp [rw ] + 1 <= io_allowed ) {
842+ if (io_allowed > 0 && tg -> io_disp [rw ] + 1 <= io_allowed )
842843 return 0 ;
843- }
844844
845845 /* Calc approx time to dispatch */
846846 jiffy_wait = jiffy_elapsed_rnd - jiffy_elapsed ;
@@ -851,7 +851,8 @@ static unsigned long tg_within_bps_limit(struct throtl_grp *tg, struct bio *bio,
851851 u64 bps_limit )
852852{
853853 bool rw = bio_data_dir (bio );
854- u64 bytes_allowed , extra_bytes ;
854+ long long bytes_allowed ;
855+ u64 extra_bytes ;
855856 unsigned long jiffy_elapsed , jiffy_wait , jiffy_elapsed_rnd ;
856857 unsigned int bio_size = throtl_bio_data_size (bio );
857858
@@ -869,9 +870,8 @@ static unsigned long tg_within_bps_limit(struct throtl_grp *tg, struct bio *bio,
869870 jiffy_elapsed_rnd = roundup (jiffy_elapsed_rnd , tg -> td -> throtl_slice );
870871 bytes_allowed = calculate_bytes_allowed (bps_limit , jiffy_elapsed_rnd ) +
871872 tg -> carryover_bytes [rw ];
872- if (tg -> bytes_disp [rw ] + bio_size <= bytes_allowed ) {
873+ if (bytes_allowed > 0 && tg -> bytes_disp [rw ] + bio_size <= bytes_allowed )
873874 return 0 ;
874- }
875875
876876 /* Calc approx time to dispatch */
877877 extra_bytes = tg -> bytes_disp [rw ] + bio_size - bytes_allowed ;
0 commit comments