test: Add warning level logging to the test framework

We add another level of logging between ERRORS and INFO/DEBUG because we print skipped tests with INFO level logging and they don't appear in CI. Reporting if they're skipped or not is useful when trying to understand if the CI is running the correct set of tests or not on a given hardware. INFO level debugging is too heavy for CI output because it prints every tensor value mismatch if there are failing tests and this could be miliions, thus it's not a good option.

After this commit, we'll also change test skip reports to warning level in various places and also change the CI to log at warning level instead of error level.

Change-Id: I4de2859b6987778d9b228fd96c2336c4ce056079
Signed-off-by: Gunes Bayir <gunes.bayir@arm.com>

Author: gunes-arm

tests/framework/Asserts.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2022, 2025 Arm Limited.
+ * Copyright (c) 2017-2022, 2025-2026 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -68,13 +68,28 @@ inline void ARM_COMPUTE_PRINT_INFO()
     arm_compute::test::framework::Framework::get().clear_test_info();
 }
 
+inline void ARM_COMPUTE_PRINT_WARNING()
+{
+    std::stringstream msg;
+    arm_compute::test::framework::Framework::get().print_test_warning(msg);
+    arm_compute::test::framework::Framework::get().log_warning(msg.str());
+    arm_compute::test::framework::Framework::get().clear_test_warning();
+}
+
 #define ARM_COMPUTE_TEST_INFO(INFO)                                               \
     {                                                                             \
         std::stringstream info;                                                   \
         info << INFO;                                                             \
         arm_compute::test::framework::Framework::get().add_test_info(info.str()); \
     }
 
+#define ARM_COMPUTE_TEST_WARNING(WARNING)                                               \
+    {                                                                                   \
+        std::stringstream warning;                                                      \
+        warning << WARNING;                                                             \
+        arm_compute::test::framework::Framework::get().add_test_warning(warning.str()); \
+    }
+
 namespace detail
 {
 #define ARM_COMPUTE_TEST_COMP_FACTORY(SEVERITY, SEVERITY_NAME, COMP, COMP_NAME, ERROR_CALL)                            \

tests/framework/Exceptions.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2025 Arm Limited.
+ * Copyright (c) 2017, 2025-2026 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -36,8 +36,13 @@ namespace framework
 LogLevel log_level_from_name(const std::string &name)
 {
     static const std::map<std::string, LogLevel> levels = {
-        {"none", LogLevel::NONE},     {"config", LogLevel::CONFIG}, {"tests", LogLevel::TESTS},
-        {"errors", LogLevel::ERRORS}, {"debug", LogLevel::DEBUG},   {"measurements", LogLevel::MEASUREMENTS},
+        {"none", LogLevel::NONE},
+        {"config", LogLevel::CONFIG},
+        {"tests", LogLevel::TESTS},
+        {"errors", LogLevel::ERRORS},
+        {"warnings", LogLevel::WARNINGS},
+        {"debug", LogLevel::DEBUG},
+        {"measurements", LogLevel::MEASUREMENTS},
         {"all", LogLevel::ALL},
     };
 
@@ -75,6 +80,9 @@ ::std::ostream &operator<<(::std::ostream &stream, LogLevel level)
         case LogLevel::ERRORS:
             stream << "ERRORS";
             break;
+        case LogLevel::WARNINGS:
+            stream << "WARNINGS";
+            break;
         case LogLevel::DEBUG:
             stream << "DEBUG";
             break;

tests/framework/Exceptions.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2018, 2025 Arm Limited.
+ * Copyright (c) 2017-2018, 2025-2026 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -42,6 +42,7 @@ namespace framework
  * NONE == Only for filtering. Not used to tag information.
  * CONFIG == Configuration info.
  * TESTS == Information about the tests.
+ * WARNINGS == Warnings, such as skipped tests.
  * ERRORS == Violated assertions/expectations.
  * DEBUG == More violated assertions/expectations.
  * MEASUREMENTS == Information about measurements.
@@ -53,6 +54,7 @@ enum class LogLevel
     CONFIG,
     TESTS,
     ERRORS,
+    WARNINGS,
     DEBUG,
     MEASUREMENTS,
     ALL,

tests/framework/Framework.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2021, 2023-2025 Arm Limited.
+ * Copyright (c) 2017-2021, 2023-2026 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -230,6 +230,34 @@ void Framework::print_test_info(std::ostream &os) const
     }
 }
 
+void Framework::add_test_warning(std::string warning)
+{
+    _test_warning.emplace_back(std::move(warning));
+}
+
+void Framework::clear_test_warning()
+{
+    _test_warning.clear();
+}
+
+bool Framework::has_test_warning() const
+{
+    return !_test_warning.empty();
+}
+
+void Framework::print_test_warning(std::ostream &os) const
+{
+    if (!_test_warning.empty())
+    {
+        os << "CONTEXT:\n";
+
+        for (const auto &str : _test_warning)
+        {
+            os << "    " << str << "\n";
+        }
+    }
+}
+
 template <typename F>
 void Framework::func_on_all_printers(F &&func)
 {
@@ -290,6 +318,14 @@ void Framework::log_info(const std::string &info)
     }
 }
 
+void Framework::log_warning(const std::string &warning)
+{
+    if (_log_level >= LogLevel::WARNINGS)
+    {
+        func_on_all_printers([&](Printer *p) { p->print_warning(warning); });
+    }
+}
+
 int Framework::num_iterations() const
 {
     return _num_iterations;

tests/framework/Framework.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2021, 2023-2025 Arm Limited.
+ * Copyright (c) 2017-2021, 2023-2026 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -176,6 +176,27 @@ class Framework final
      */
     void print_test_info(std::ostream &os) const;
 
+    /** Add warning string for the next expectation/assertion.
+     *
+     * @param[in] info Warning string.
+     */
+    void add_test_warning(std::string warning);
+
+    /** Clear the collected test warning. */
+    void clear_test_warning();
+
+    /** Check if any warning has been registered.
+     *
+     * @return True if there is test warning.
+     */
+    bool has_test_warning() const;
+
+    /** Print test warning.
+     *
+     * @param[out] os Output stream.
+     */
+    void print_test_warning(std::ostream &os) const;
+
     /** Tell the framework that execution of a test starts.
      *
      * @param[in] info Test info.
@@ -206,6 +227,12 @@ class Framework final
      */
     void log_info(const std::string &info);
 
+    /** Print the warnings that has already been logged
+     *
+     * @param[in] info Description of the log warning.
+     */
+    void log_warning(const std::string &warning);
+
     /** Number of iterations per test case.
      *
      * @return Number of iterations per test case.
@@ -406,6 +433,7 @@ class Framework final
     const TestInfo             *_current_test_info{nullptr};
     TestResult                 *_current_test_result{nullptr};
     std::vector<std::string>    _test_info{};
+    std::vector<std::string>    _test_warning{};
 };
 
 template <typename T>

tests/framework/command_line/CommonOptions.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018-2020,2024-2025 Arm Limited.
+ * Copyright (c) 2018-2020,2024-2026 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -74,8 +74,8 @@ CommonOptions::CommonOptions(CommandLineParser &parser)
     };
 
     std::set<LogLevel> supported_log_levels{
-        LogLevel::NONE,  LogLevel::CONFIG,       LogLevel::TESTS, LogLevel::ERRORS,
-        LogLevel::DEBUG, LogLevel::MEASUREMENTS, LogLevel::ALL,
+        LogLevel::NONE,     LogLevel::CONFIG, LogLevel::TESTS,        LogLevel::ERRORS,
+        LogLevel::WARNINGS, LogLevel::DEBUG,  LogLevel::MEASUREMENTS, LogLevel::ALL,
     };
 
     instruments = parser.add_option<EnumListOption<InstrumentsDescription>>(

tests/framework/printers/JSONPrinter.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2019,2021, 2025 Arm Limited.
+ * Copyright (c) 2017-2019,2021, 2025-2026 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -132,6 +132,7 @@ void JSONPrinter::print_errors_header()
 {
     _errors.clear();
     _expected_errors.clear();
+    _warnings.clear();
     _infos.clear();
 }
 
@@ -147,6 +148,10 @@ void JSONPrinter::print_errors_footer()
     print_strings(_expected_errors.begin(), _expected_errors.end());
     *_stream << "]";
 
+    *_stream << R"(, "warnings" : [)";
+    print_strings(_warnings.begin(), _warnings.end());
+    *_stream << "]";
+
     *_stream << R"(, "infos" : [)";
     print_strings(_infos.begin(), _infos.end());
     *_stream << "]";
@@ -164,6 +169,11 @@ void JSONPrinter::print_error(const std::exception &error, bool expected)
     }
 }
 
+void JSONPrinter::print_warning(const std::string &warning)
+{
+    _warnings.push_back(warning);
+}
+
 void JSONPrinter::print_info(const std::string &info)
 {
     _infos.push_back(info);

tests/framework/printers/JSONPrinter.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017,2021, 2025 Arm Limited.
+ * Copyright (c) 2017,2021, 2025-2026 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -49,6 +49,7 @@ class JSONPrinter : public Printer
     void print_errors_header() override;
     void print_errors_footer() override;
     void print_error(const std::exception &error, bool expected) override;
+    void print_warning(const std::string &warning) override;
     void print_info(const std::string &info) override;
     void print_profiler_header(const std::string &header_data) override;
     void print_measurements(const Profiler::MeasurementsMap &measurements) override;
@@ -60,6 +61,7 @@ class JSONPrinter : public Printer
     void print_strings(T &&first, T &&last);
 
     std::list<std::string> _infos{};
+    std::list<std::string> _warnings{};
     std::list<std::string> _errors{};
     std::list<std::string> _expected_errors{};
 

tests/framework/printers/PrettyPrinter.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2019,2021, 2025 Arm Limited.
+ * Copyright (c) 2017-2019,2021, 2025-2026 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -102,6 +102,11 @@ void PrettyPrinter::print_info(const std::string &info)
     *_stream << begin_color("1") << "INFO: " << info << end_color() << "\n";
 }
 
+void PrettyPrinter::print_warning(const std::string &warning)
+{
+    *_stream << begin_color("1") << "WARNING: " << warning << end_color() << "\n";
+}
+
 void PrettyPrinter::print_error(const std::exception &error, bool expected)
 {
     std::string prefix = expected ? "EXPECTED ERROR: " : "ERROR: ";

tests/framework/printers/PrettyPrinter.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017,2021, 2025 Arm Limited.
+ * Copyright (c) 2017,2021, 2025-2026 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -54,6 +54,7 @@ class PrettyPrinter : public Printer
     void print_errors_header() override;
     void print_errors_footer() override;
     void print_error(const std::exception &error, bool expected) override;
+    void print_warning(const std::string &warning) override;
     void print_info(const std::string &info) override;
     void print_profiler_header(const std::string &header_data) override;
     void print_measurements(const Profiler::MeasurementsMap &measurements) override;