Loosen tolerances for text-only image differences

Author: ayshih

lib/matplotlib/tests/test_axes.py

@@ -239,8 +239,9 @@ def test_matshow(fig_test, fig_ref):
     ax_ref.xaxis.set_ticks_position('both')
 
 
+# TODO: tighten tolerance after baseline image is regenerated for text overhaul
 @image_comparison([f'formatter_ticker_{i:03d}.png' for i in range(1, 6)],
-                  tol=0 if platform.machine() == 'x86_64' else 0.031)
+                  tol=0.02 if platform.machine() == 'x86_64' else 0.04)
 def test_formatter_ticker():
     import matplotlib.testing.jpl_units as units
     units.register()
@@ -809,7 +810,8 @@ def test_annotate_signature():
         assert p1 == p2
 
 
-@image_comparison(['fill_units.png'], savefig_kwarg={'dpi': 60})
+# TODO: tighten tolerance after baseline image is regenerated for text overhaul
+@image_comparison(['fill_units.png'], savefig_kwarg={'dpi': 60}, tol=0.2)
 def test_fill_units():
     import matplotlib.testing.jpl_units as units
     units.register()
@@ -1516,7 +1518,8 @@ def test_pcolormesh_log_scale(fig_test, fig_ref):
     ax.set_xscale('log')
 
 
-@image_comparison(['pcolormesh_datetime_axis.png'], style='mpl20')
+# TODO: tighten tolerance after baseline image is regenerated for text overhaul
+@image_comparison(['pcolormesh_datetime_axis.png'], style='mpl20', tol=0.3)
 def test_pcolormesh_datetime_axis():
     # Remove this line when this test image is regenerated.
     plt.rcParams['pcolormesh.snap'] = False
@@ -1544,7 +1547,8 @@ def test_pcolormesh_datetime_axis():
             label.set_rotation(30)
 
 
-@image_comparison(['pcolor_datetime_axis.png'], style='mpl20')
+# TODO: tighten tolerance after baseline image is regenerated for text overhaul
+@image_comparison(['pcolor_datetime_axis.png'], style='mpl20', tol=0.3)
 def test_pcolor_datetime_axis():
     fig = plt.figure()
     fig.subplots_adjust(hspace=0.4, top=0.98, bottom=.15)
@@ -2762,7 +2766,8 @@ def test_stairs_options():
     ax.legend(loc=0)
 
 
-@image_comparison(['test_stairs_datetime.png'])
+# TODO: tighten tolerance after baseline image is regenerated for text overhaul
+@image_comparison(['test_stairs_datetime.png'], tol=0.2)
 def test_stairs_datetime():
     f, ax = plt.subplots(constrained_layout=True)
     ax.stairs(np.arange(36),
@@ -6501,7 +6506,8 @@ def test_pie_frame_grid():
     plt.axis('equal')
 
 
-@image_comparison(['pie_rotatelabels_true.png'], style='mpl20', tol=0.009)
+# TODO: tighten tolerance after baseline image is regenerated for text overhaul
+@image_comparison(['pie_rotatelabels_true.png'], style='mpl20', tol=0.1)
 def test_pie_rotatelabels_true():
     # The slices will be ordered and plotted counter-clockwise.
     labels = 'Hogwarts', 'Frogs', 'Dogs', 'Logs'

lib/matplotlib/tests/test_colorbar.py

@@ -152,14 +152,15 @@ def test_colorbar_extension_inverted_axis(orientation, extend, expected):
         assert len(cbar._extend_patches) == 1
 
 
+# TODO: tighten tolerance after baseline image is regenerated for text overhaul
 @pytest.mark.parametrize('use_gridspec', [True, False])
 @image_comparison(['cbar_with_orientation',
                    'cbar_locationing',
                    'double_cbar',
                    'cbar_sharing',
                    ],
                   extensions=['png'], remove_text=True,
-                  savefig_kwarg={'dpi': 40})
+                  savefig_kwarg={'dpi': 40}, tol=0.05)
 def test_colorbar_positioning(use_gridspec):
     # Remove this line when this test image is regenerated.
     plt.rcParams['pcolormesh.snap'] = False
@@ -731,7 +732,8 @@ def test_colorbar_label():
     assert cbar3.ax.get_xlabel() == 'horizontal cbar'
 
 
-@image_comparison(['colorbar_keeping_xlabel.png'], style='mpl20')
+# TODO: tighten tolerance after baseline image is regenerated for text overhaul
+@image_comparison(['colorbar_keeping_xlabel.png'], style='mpl20', tol=0.03)
 def test_keeping_xlabel():
     # github issue #23398 - xlabels being ignored in colorbar axis
     arr = np.arange(25).reshape((5, 5))

lib/matplotlib/tests/test_contour.py

@@ -127,8 +127,9 @@ def test_contour_manual_moveto():
     assert clabels[0].get_text() == "0"
 
 
+# TODO: tighten tolerance after baseline image is regenerated for text overhaul
 @image_comparison(['contour_disconnected_segments'],
-                  remove_text=True, style='mpl20', extensions=['png'])
+                  remove_text=True, style='mpl20', extensions=['png'], tol=0.01)
 def test_contour_label_with_disconnected_segments():
     x, y = np.mgrid[-1:1:21j, -1:1:21j]
     z = 1 / np.sqrt(0.01 + (x + 0.3) ** 2 + y ** 2)
@@ -229,7 +230,8 @@ def test_lognorm_levels(n_levels):
     assert len(visible_levels) <= n_levels + 1
 
 
-@image_comparison(['contour_datetime_axis.png'], style='mpl20')
+# TODO: tighten tolerance after baseline image is regenerated for text overhaul
+@image_comparison(['contour_datetime_axis.png'], style='mpl20', tol=0.3)
 def test_contour_datetime_axis():
     fig = plt.figure()
     fig.subplots_adjust(hspace=0.4, top=0.98, bottom=.15)

lib/matplotlib/tests/test_dates.py

@@ -152,7 +152,8 @@ def test_date_axhspan():
     fig.subplots_adjust(left=0.25)
 
 
-@image_comparison(['date_axvspan.png'])
+# TODO: tighten tolerance after baseline image is regenerated for text overhaul
+@image_comparison(['date_axvspan.png'], tol=0.07)
 def test_date_axvspan():
     # test axvspan with date inputs
     t0 = datetime.datetime(2000, 1, 20)
@@ -176,7 +177,8 @@ def test_date_axhline():
     fig.subplots_adjust(left=0.25)
 
 
-@image_comparison(['date_axvline.png'])
+# TODO: tighten tolerance after baseline image is regenerated for text overhaul
+@image_comparison(['date_axvline.png'], tol=0.09)
 def test_date_axvline():
     # test axvline with date inputs
     t0 = datetime.datetime(2000, 1, 20)
@@ -226,7 +228,8 @@ def wrapper():
     return wrapper
 
 
-@image_comparison(['RRuleLocator_bounds.png'])
+# TODO: tighten tolerance after baseline image is regenerated for text overhaul
+@image_comparison(['RRuleLocator_bounds.png'], tol=0.07)
 def test_RRuleLocator():
     import matplotlib.testing.jpl_units as units
     units.register()
@@ -270,7 +273,8 @@ def test_RRuleLocator_close_minmax():
     assert list(map(str, mdates.num2date(loc.tick_values(d1, d2)))) == expected
 
 
-@image_comparison(['DateFormatter_fractionalSeconds.png'])
+# TODO: tighten tolerance after baseline image is regenerated for text overhaul
+@image_comparison(['DateFormatter_fractionalSeconds.png'], tol=0.11)
 def test_DateFormatter():
     import matplotlib.testing.jpl_units as units
     units.register()

lib/matplotlib/tests/test_figure.py

@@ -25,8 +25,9 @@
 import matplotlib.dates as mdates
 
 
+# TODO: tighten tolerance after baseline image is regenerated for text overhaul
 @image_comparison(['figure_align_labels'], extensions=['png', 'svg'],
-                  tol=0 if platform.machine() == 'x86_64' else 0.01)
+                  tol=0.1 if platform.machine() == 'x86_64' else 0.1)
 def test_align_labels():
     fig = plt.figure(layout='tight')
     gs = gridspec.GridSpec(3, 3)
@@ -66,9 +67,10 @@ def test_align_labels():
     fig.align_labels()
 
 
+# TODO: tighten tolerance after baseline image is regenerated for text overhaul
 @image_comparison(['figure_align_titles_tight.png',
                    'figure_align_titles_constrained.png'],
-                  tol=0 if platform.machine() == 'x86_64' else 0.022,
+                  tol=0.3 if platform.machine() == 'x86_64' else 0.04,
                   style='mpl20')
 def test_align_titles():
     for layout in ['tight', 'constrained']:
@@ -320,7 +322,8 @@ def test_add_subplot_invalid():
         fig.add_subplot(ax)
 
 
-@image_comparison(['figure_suptitle.png'])
+# TODO: tighten tolerance after baseline image is regenerated for text overhaul
+@image_comparison(['figure_suptitle.png'], tol=0.02)
 def test_suptitle():
     fig, _ = plt.subplots()
     fig.suptitle('hello', color='r')
@@ -1396,8 +1399,9 @@ def test_subfigure_ss():
     fig.suptitle('Figure suptitle', fontsize='xx-large')
 
 
+# TODO: tighten tolerance after baseline image is regenerated for text overhaul
 @image_comparison(['test_subfigure_double.png'], style='mpl20',
-                  savefig_kwarg={'facecolor': 'teal'})
+                  savefig_kwarg={'facecolor': 'teal'}, tol=0.02)
 def test_subfigure_double():
     # test assigning the subfigure via subplotspec
     np.random.seed(19680801)

lib/matplotlib/tests/test_polar.py

@@ -214,7 +214,8 @@ def test_polar_theta_position():
     ax.set_theta_direction('clockwise')
 
 
-@image_comparison(['polar_rlabel_position.png'], style='default')
+# TODO: tighten tolerance after baseline image is regenerated for text overhaul
+@image_comparison(['polar_rlabel_position.png'], style='default', tol=0.07)
 def test_polar_rlabel_position():
     fig = plt.figure()
     ax = fig.add_subplot(projection='polar')
@@ -229,7 +230,8 @@ def test_polar_title_position():
     ax.set_title('foo')
 
 
-@image_comparison(['polar_theta_wedge.png'], style='default')
+# TODO: tighten tolerance after baseline image is regenerated for text overhaul
+@image_comparison(['polar_theta_wedge.png'], style='default', tol=0.2)
 def test_polar_theta_limits():
     r = np.arange(0, 3.0, 0.01)
     theta = 2*np.pi*r

lib/matplotlib/tests/test_text.py

@@ -139,7 +139,8 @@ def test_multiline():
     ax.set_yticks([])
 
 
-@image_comparison(['multiline2'], style='mpl20')
+# TODO: tighten tolerance after baseline image is regenerated for text overhaul
+@image_comparison(['multiline2'], style='mpl20', tol=0.05)
 def test_multiline2():
     # Remove this line when this test image is regenerated.
     plt.rcParams['text.kerning_factor'] = 6
@@ -210,7 +211,8 @@ def test_antialiasing():
     mpl.rcParams['text.antialiased'] = False  # Should not affect existing text.
 
 
-@image_comparison(['text_contains.png'])
+# TODO: tighten tolerance after baseline image is regenerated for text overhaul
+@image_comparison(['text_contains.png'], tol=0.05)
 def test_contains():
     fig = plt.figure()
     ax = plt.axes()
@@ -279,7 +281,8 @@ def test_titles():
     ax.set_yticks([])
 
 
-@image_comparison(['text_alignment'], style='mpl20')
+# TODO: tighten tolerance after baseline image is regenerated for text overhaul
+@image_comparison(['text_alignment'], style='mpl20', tol=0.08)
 def test_alignment():
     plt.figure()
     ax = plt.subplot(1, 1, 1)
@@ -1132,8 +1135,9 @@ def test_empty_annotation_get_window_extent():
     assert points[0, 1] == 50.0
 
 
+# TODO: tighten tolerance after baseline image is regenerated for text overhaul
 @image_comparison(baseline_images=['basictext_wrap'],
-                  extensions=['png'])
+                  extensions=['png'], tol=0.3)
 def test_basic_wrap():
     fig = plt.figure()
     plt.axis([0, 10, 0, 10])
@@ -1149,8 +1153,9 @@ def test_basic_wrap():
     plt.text(-1, 0, t, ha='left', rotation=-15, wrap=True)
 
 
+# TODO: tighten tolerance after baseline image is regenerated for text overhaul
 @image_comparison(baseline_images=['fonttext_wrap'],
-                  extensions=['png'])
+                  extensions=['png'], tol=0.3)
 def test_font_wrap():
     fig = plt.figure()
     plt.axis([0, 10, 0, 10])
@@ -1182,8 +1187,9 @@ def test_va_for_angle():
         assert alignment in ['center', 'top', 'baseline']
 
 
+# TODO: tighten tolerance after baseline image is regenerated for text overhaul
 @image_comparison(baseline_images=['xtick_rotation_mode'],
-                  remove_text=False, extensions=['png'], style='mpl20')
+                  remove_text=False, extensions=['png'], style='mpl20', tol=0.3)
 def test_xtick_rotation_mode():
     fig, ax = plt.subplots(figsize=(12, 1))
     ax.set_yticks([])
@@ -1202,8 +1208,9 @@ def test_xtick_rotation_mode():
     plt.subplots_adjust(left=0.01, right=0.99, top=.6, bottom=.4)
 
 
+# TODO: tighten tolerance after baseline image is regenerated for text overhaul
 @image_comparison(baseline_images=['ytick_rotation_mode'],
-                  remove_text=False, extensions=['png'], style='mpl20')
+                  remove_text=False, extensions=['png'], style='mpl20', tol=0.3)
 def test_ytick_rotation_mode():
     fig, ax = plt.subplots(figsize=(1, 12))
     ax.set_xticks([])

lib/matplotlib/tests/test_units.py

@@ -80,8 +80,9 @@ def default_units(value, axis):
 
 # Tests that the conversion machinery works properly for classes that
 # work as a facade over numpy arrays (like pint)
+# TODO: tighten tolerance after baseline image is regenerated for text overhaul
 @image_comparison(['plot_pint.png'], style='mpl20',
-                  tol=0 if platform.machine() == 'x86_64' else 0.03)
+                  tol=0.03 if platform.machine() == 'x86_64' else 0.04)
 def test_numpy_facade(quantity_converter):
     # use former defaults to match existing baseline image
     plt.rcParams['axes.formatter.limits'] = -7, 7
@@ -142,8 +143,9 @@ def test_jpl_bar_units():
     ax.set_ylim([b - 1 * day, b + w[-1] + (1.001) * day])
 
 
+# TODO: tighten tolerance after baseline image is regenerated for text overhaul
 @image_comparison(['jpl_barh_units.png'],
-                  savefig_kwarg={'dpi': 120}, style='mpl20')
+                  savefig_kwarg={'dpi': 120}, style='mpl20', tol=0.02)
 def test_jpl_barh_units():
     import matplotlib.testing.jpl_units as units
     units.register()

lib/matplotlib/tests/test_usetex.py

@@ -226,8 +226,9 @@ def test_pdf_type1_font_subsetting():
     _old_gs_version = True
 
 
+# TODO: tighten tolerance after baseline image is regenerated for text overhaul
 @image_comparison(baseline_images=['rotation'], extensions=['eps', 'pdf', 'png', 'svg'],
-                  style='mpl20', tol=3.91 if _old_gs_version else 0)
+                  style='mpl20', tol=3.91 if _old_gs_version else 0.2)
 def test_rotation():
     mpl.rcParams['text.usetex'] = True
 

lib/mpl_toolkits/axisartist/tests/test_axis_artist.py

@@ -24,7 +24,8 @@ def test_ticks():
     ax.add_artist(ticks_out)
 
 
-@image_comparison(['axis_artist_labelbase.png'], style='default')
+# TODO: tighten tolerance after baseline image is regenerated for text overhaul
+@image_comparison(['axis_artist_labelbase.png'], style='default', tol=0.02)
 def test_labelbase():
     # Remove this line when this test image is regenerated.
     plt.rcParams['text.kerning_factor'] = 6
@@ -41,7 +42,8 @@ def test_labelbase():
     ax.add_artist(label)
 
 
-@image_comparison(['axis_artist_ticklabels.png'], style='default')
+# TODO: tighten tolerance after baseline image is regenerated for text overhaul
+@image_comparison(['axis_artist_ticklabels.png'], style='default', tol=0.03)
 def test_ticklabels():
     # Remove this line when this test image is regenerated.
     plt.rcParams['text.kerning_factor'] = 6
@@ -76,7 +78,8 @@ def test_ticklabels():
     ax.set_ylim(0, 1)
 
 
-@image_comparison(['axis_artist.png'], style='default')
+# TODO: tighten tolerance after baseline image is regenerated for text overhaul
+@image_comparison(['axis_artist.png'], style='default', tol=0.03)
 def test_axis_artist():
     # Remove this line when this test image is regenerated.
     plt.rcParams['text.kerning_factor'] = 6