From 6c46d48fe5a37724ead50ef8e317fa9fb20520e9 Mon Sep 17 00:00:00 2001 From: Tarun Wadhwa Date: Tue, 16 Jun 2026 18:10:20 +0000 Subject: [PATCH 01/14] WIP: MDEV-37605 convert innodb binlog to legacy --- client/mysqlbinlog.cc | 533 +++++++++++++++++- ...lbinlog_convert_engine_binlog_basic.result | 25 + ...sqlbinlog_convert_engine_binlog_basic.test | 61 ++ ...nvert_engine_binlog_gtid_list_event.result | 46 ++ ...convert_engine_binlog_gtid_list_event.test | 129 +++++ sql/log_event.cc | 93 +++ sql/log_event.h | 11 +- sql/log_event_client.cc | 18 + sql/log_event_server.cc | 90 --- sql/rpl_gtid.cc | 9 +- 10 files changed, 900 insertions(+), 115 deletions(-) create mode 100644 mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.result create mode 100644 mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.test create mode 100644 mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_gtid_list_event.result create mode 100644 mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_gtid_list_event.test diff --git a/client/mysqlbinlog.cc b/client/mysqlbinlog.cc index a286e99203e6d..3de59f34bc0dc 100644 --- a/client/mysqlbinlog.cc +++ b/client/mysqlbinlog.cc @@ -128,6 +128,7 @@ static my_bool print_row_count_used= 0, print_row_event_positions_used= 0; static my_bool debug_info_flag, debug_check_flag; static my_bool force_if_open_opt= 1; static my_bool opt_raw_mode= 0, opt_stop_never= 0; +static my_bool opt_convert_engine_binlog= 0; my_bool opt_gtid_strict_mode= true; static ulong opt_stop_never_slave_server_id= 0; static my_bool opt_verify_binlog_checksum= 1; @@ -211,6 +212,17 @@ static enum Binlog_format { static handler_binlog_reader *engine_binlog_reader; +static FILE *output_legacy_binlog_file= 0; +static ulonglong convert_engine_output_index= 0; +static char default_output_legacy_binlog_prefix[]= "legacy_log"; + +static char out_file_name[FN_REFLEN + 1]= {0}; + +/* Used to track the position of the log file for computing legacy end_log_pos for converted legacy binlog file */ +static ulonglong log_file_pos= 0; + +static rpl_binlog_state_base *gtid_state= NULL; + /** Pointer to the last read Annotate_rows_log_event. Having read an @@ -1556,6 +1568,438 @@ Exit_status process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev, DBUG_RETURN(retval); } +/* + ############################################################################## + Helper functions used for serializing the events to the legacy binlog format. + ############################################################################## +*/ + +static bool write_event_header(FILE *outfile, Log_event_type event_type, + ulong extra_len, time_t timestamp, + my_bool *do_checksum, ha_checksum *crc, + enum_binlog_checksum_alg checksum_alg) +{ + uchar header[LOG_EVENT_HEADER_LEN]; + ulong event_len; + + *do_checksum= checksum_alg != BINLOG_CHECKSUM_ALG_OFF && + checksum_alg != BINLOG_CHECKSUM_ALG_UNDEF; + + int4store(header, timestamp); + header[EVENT_TYPE_OFFSET] = (uchar)event_type; + event_len= LOG_EVENT_HEADER_LEN + extra_len + + (*do_checksum ? BINLOG_CHECKSUM_LEN : 0); + + // TODO: Tarun get this reviewed. what should the server id be? should i get from + // innodb binlog header + int4store(header + SERVER_ID_OFFSET, 0); + int4store(header + EVENT_LEN_OFFSET, event_len); + /* + Notes: For a normal open/current binlog file, the format-description + header flags are typically 0x0001. After clean close, they become + 0x0000. + For GTID_LIST_EVENT and BINLOG_CHECKPOINT_EVENT, the flags are 0x0000 typically. + TODO: Tarun get this reviewed. + */ + int2store(header + FLAGS_OFFSET, 0); + /* Update the log_file_pos */ + log_file_pos+= event_len; + int4store(header + LOG_POS_OFFSET, log_file_pos); + /* Write this header to outfile */ + if (my_fwrite(outfile, (const uchar *)header, LOG_EVENT_HEADER_LEN, MYF(MY_NABP))) { + error("Could not write header into converted binlog file '%s'", + out_file_name); + return true; + } + if (*do_checksum) + { + *crc= my_checksum(0, (uchar*)header, sizeof(header)); + } + return false; +} + + +static bool +write_event_footer(FILE *outfile, my_bool do_checksum, ha_checksum crc) +{ + if (do_checksum) + { + char b[BINLOG_CHECKSUM_LEN]; + int4store(b, crc); + if (my_fwrite(outfile, (const uchar *)b, sizeof(b), MYF(MY_NABP))) + { + error("Could not write footer into converted binlog file '%s'", + out_file_name); + return true; + } + } + return false; +} + +/* + updates the log_file_pos(end_log_pos) to include the raw bytes of the event + It does not update the log_pos variable of ev as it is not + really needed for the conversion. +*/ +static void update_event_end_log_pos(Log_event *ev) +{ + DBUG_ASSERT(ev->temp_buf != NULL && ev->data_written >= LOG_EVENT_HEADER_LEN); + log_file_pos+= ev->data_written; + int4store(&ev->temp_buf[LOG_POS_OFFSET], log_file_pos); +} + +/* + Updates the checksum of the event only in the raw bytes of the event. + Mainly used for FORMAT_DESCRIPTION_EVENT. +*/ +static void update_checksum(Log_event *ev) +{ + DBUG_ASSERT(ev->temp_buf != NULL && + ev->data_written >= LOG_EVENT_HEADER_LEN + BINLOG_CHECKSUM_LEN); + ha_checksum crc= my_checksum(0, (uchar *) ev->temp_buf, + ev->data_written - BINLOG_CHECKSUM_LEN); + int4store(&ev->temp_buf[ev->data_written - BINLOG_CHECKSUM_LEN], crc); +} + +static bool write_format_description_event_to_legacy_binlog( + FILE *outfile, Format_description_log_event *fdev) +{ + // temp_buf stores the raw bytes of the event and data_written is the length of those raw bytes + if(fdev->temp_buf) { + /* Update the log_file_pos */ + update_event_end_log_pos(fdev); + + /* recompute checksum */ + update_checksum(fdev); + + if (my_fwrite(outfile, (const uchar *)fdev->temp_buf, + fdev->data_written, MYF(MY_NABP))) + { + error("Could not write into converted binlog file '%s'", + out_file_name); + return true; + } + + fflush(outfile); + + return false; + } + + /* + fdev->temp_buf is empty, which means fdev was dynamically generated. + Therefore, serialize the FDEV and write it to the output file. + */ + + my_bool do_checksum; + ha_checksum crc; + + char buf[320]; + String str(buf, sizeof(buf), system_charset_info); + + str.length(0); + fdev->dont_set_created= true; + fdev->used_checksum_alg= BINLOG_CHECKSUM_ALG_OFF; + + if (fdev->to_packet(&str)) + { + error("Failed due to out-of-memory writing Format_description event"); + return true; + } + /* Write header of FORMAT_DESCRIPTION_EVENT to output legacy binlog file first */ + if (write_event_header(outfile, FORMAT_DESCRIPTION_EVENT, str.length(), + fdev->created, &do_checksum, &crc, + BINLOG_CHECKSUM_ALG_CRC32)) + { + error("Could not write FORMAT_DESCRIPTION_EVENT header to output legacy " + "binlog file"); + return true; + } + + /* Write body to output legacy binlog file */ + if (my_fwrite(outfile, (const uchar *) str.ptr(), str.length(), + MYF(MY_NABP))) + { + error("Could not write body into converted binlog file '%s'", + out_file_name); + return true; + } + + if (do_checksum) + { + crc= my_checksum(crc, (uchar *) str.ptr(), str.length()); + } + + /* Write footer to output legacy binlog file */ + if (write_event_footer(outfile, do_checksum, crc)) + { + error("Could not write footer into converted binlog file '%s'", + out_file_name); + return true; + } + fflush(outfile); + + return false; +} + +static bool write_gtid_list_event_to_legacy_binlog(FILE *outfile, + Gtid_list_log_event *glev) +{ + my_bool do_checksum; + ha_checksum crc= 0; + char buf[128]; + String str(buf, sizeof(buf), system_charset_info); + str.length(0); + + if (glev->to_packet(&str)) + { + error("Failed due to out-of-memory writing Gtid_list event"); + return true; + } + + /* Write header of GTID_LIST_EVENT to output legacy binlog file first */ + /* TODO: Tarun verify and fix behaviour of timestamp (ts) */ + time_t ts= 0; + if (write_event_header(outfile, GTID_LIST_EVENT, str.length(), ts, + &do_checksum, &crc, BINLOG_CHECKSUM_ALG_OFF)) + { + error( + "Could not write GTID_LIST_EVENT header to output legacy binlog file"); + return true; + } + + /* Write body to output legacy binlog file */ + if (my_fwrite(outfile, (const uchar *)str.ptr(), str.length(), MYF(MY_NABP))) { + error("Could not write body into converted binlog file '%s'", + out_file_name); + return true; + } + + /* + TODO: Tarun get this reviewed. We are disabling/not supporting the checksum in every event. + Should I keep the below code for extensibility? + (do_checksum will always be false here) + */ + if (do_checksum) { + crc= my_checksum(crc, (uchar*)str.ptr(), str.length()); + } + + /* Write footer to output legacy binlog file */ + if (write_event_footer(outfile, do_checksum, crc)) { + error("Could not write footer into converted binlog file '%s'", + out_file_name); + return true; + } + fflush(outfile); + + return false; +} + +static bool write_binlog_checkpoint_event_to_legacy_binlog( + FILE *outfile, Binlog_checkpoint_log_event *bcle) +{ + my_bool do_checksum; + ha_checksum crc= 0; + char buf[128]; + String str(buf, sizeof(buf), system_charset_info); + str.length(0); + + /* Generate BINLOG_CHECKPOINT_EVENT body (This is equivalent to the + to_packet() method used in case of FORMAT_DESCRIPTION_EVENT, + GTID_LIST_EVENT, etc.) */ + uchar header_buf[BINLOG_CHECKPOINT_HEADER_LEN]; + int4store(header_buf, bcle->binlog_file_len); + if (str.append((char *) header_buf, BINLOG_CHECKPOINT_HEADER_LEN)) + { + error("Failed due to out-of-memory writing BINLOG_CHECKPOINT_EVENT body"); + return true; + } + + if (str.append((char *) bcle->binlog_file_name, bcle->binlog_file_len)) + { + error("Failed due to out-of-memory writing BINLOG_CHECKPOINT_EVENT body"); + return true; + } + + /* Write header of BINLOG_CHECKPOINT_EVENT to output legacy binlog file first + */ + /* TODO: Tarun verify and fix behaviour of timestamp (ts) */ + time_t ts= 0; + if (write_event_header(outfile, BINLOG_CHECKPOINT_EVENT, str.length(), ts, + &do_checksum, &crc, BINLOG_CHECKSUM_ALG_OFF)) + { + error("Could not write BINLOG_CHECKPOINT_EVENT header to output legacy " + "binlog file"); + return true; + } + + /* Write body to output legacy binlog file */ + if (my_fwrite(outfile, (const uchar *) str.ptr(), str.length(), + MYF(MY_NABP))) + { + error("Could not write body into converted binlog file '%s'", + out_file_name); + return true; + } + + if (do_checksum) + { + crc= my_checksum(crc, (uchar *) str.ptr(), str.length()); + } + + /* write footer to output legacy binlog file */ + if (write_event_footer(outfile, do_checksum, crc)) + { + error("Could not write footer into converted binlog file '%s'", + out_file_name); + return true; + } + fflush(outfile); + + return false; +} + +static bool init_output_legacy_binlog(FILE **out_file, char *out_name, + size_t out_name_len) +{ + + /* Reset the log_file_pos to 0 for the new output legacy binlog file */ + log_file_pos= 0; + + const char *prefix= result_file_name ? result_file_name + : default_output_legacy_binlog_prefix; + + my_snprintf(out_name, out_name_len, "%s.%06llu", prefix, + ++convert_engine_output_index); + + if (!(*out_file= my_fopen(out_name, O_WRONLY | O_BINARY, MYF(MY_WME)))) + { + error("Could not create converted binlog file: %s", out_name); + return true; + } + + fprintf(stderr, "Generated output file: %s\n", out_name); + + // Write the BINLOG_MAGIC to the output legacy binlog file + if (my_fwrite(*out_file, (const uchar *) BINLOG_MAGIC, BIN_LOG_HEADER_SIZE, + MYF(MY_NABP))) + { + error("Could not write into converted binlog file '%s'", out_name); + return true; + } + log_file_pos+= BIN_LOG_HEADER_SIZE; + + // Write the FORMAT_DESCRIPTION_EVENT to the output legacy binlog file + if (write_format_description_event_to_legacy_binlog(*out_file, + glob_description_event)) + { + error("Could not write FORMAT_DESCRIPTION_EVENT to output legacy binlog " + "file"); + return true; + } + + /* Write the GTID_LIST_EVENT to the output legacy binlog file */ + Gtid_list_log_event gle= Gtid_list_log_event(gtid_state); + if (write_gtid_list_event_to_legacy_binlog(*out_file, &gle)) + { + error("Could not write GTID_LIST_EVENT to output legacy binlog file"); + return true; + } + + /* BINLOG_CHECKPOINT_EVENT only accepts the basename of the binlog file, not + * the full path */ + size_t off= dirname_length(out_name); + uint32 length= (uint32) (strlen(out_name) - off); + + // Write the BINLOG_CHECKPOINT_EVENT to the output legacy binlog file + Binlog_checkpoint_log_event bcle= + Binlog_checkpoint_log_event(out_name + off, length); + if (!bcle.is_valid()) + { + error("Failed to create BINLOG_CHECKPOINT_EVENT"); + return true; + } + if (write_binlog_checkpoint_event_to_legacy_binlog(*out_file, &bcle)) + { + error("Could not write BINLOG_CHECKPOINT_EVENT to output legacy binlog " + "file"); + return true; + } + return false; +} + +/* + Writes the event to the converted legacy binlog file + @param ev: The event to write + @return: OK_CONTINUE if successful, ERROR_STOP if failed +*/ +static Exit_status write_event_to_legacy_binlog(Log_event *ev) +{ + + // if event type is FORMAT_DESCRIPTION_EVENT, store the event in global variable glob_description_event + if (ev->get_type_code() == FORMAT_DESCRIPTION_EVENT) { + + delete glob_description_event; + glob_description_event= (Format_description_log_event*) ev; + + // close the output legacy binlog file if it is open + if (output_legacy_binlog_file) { + // TODO: tarun Write the ROTATE_EVENT to the output legacy binlog file + my_fclose(output_legacy_binlog_file, MYF(0)); + output_legacy_binlog_file= NULL; + } + + return OK_CONTINUE; + } + + /* + Open and initialize the output legacy binlog file. + Writes the FORMAT_DESCRIPTION_EVENT, GTID_LIST event, and BINLOG_CHECKPOINT event. + */ + if (!output_legacy_binlog_file) + { + + if (init_output_legacy_binlog(&output_legacy_binlog_file, out_file_name, + sizeof(out_file_name))) + goto err; + } + + /* + if event type is GTID_EVENT, update the gtid_state + which will be used to write the GTID_LIST_EVENT in + the next output legacy binlog file. + */ + if (ev->get_type_code() == GTID_EVENT) { + rpl_gtid ev_gtid; + Gtid_log_event *gle= (Gtid_log_event*) ev; + ev_gtid= {gle->domain_id, gle->server_id, gle->seq_no}; + + if (gtid_state->update_nolock(&ev_gtid)) { + error("Failed to update GTID state"); + goto err; + } + } + + /* Update the log_file_pos */ + update_event_end_log_pos(ev); + + // ev->temp_buf contains the raw event bytes and ev->data_written is the length of the event + if (my_fwrite(output_legacy_binlog_file, (const uchar *)ev->temp_buf, + ev->data_written, MYF(MY_NABP))) + { + error("Could not write into converted binlog file '%s'", + out_file_name); + goto err; + } + fflush(output_legacy_binlog_file); + + delete ev; + return OK_CONTINUE; + +err: + delete ev; + return ERROR_STOP; +} + static struct my_option my_options[] = { @@ -1609,6 +2053,10 @@ static struct my_option my_options[] = "already have. NOTE: you will need a SUPER privilege to use this option.", &disable_log_bin, &disable_log_bin, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"convert-engine-binlog", 0, + "Convert InnoDB based engine binlog files to legacy binlog files.", + &opt_convert_engine_binlog, &opt_convert_engine_binlog, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"flashback", 'B', "Flashback feature can rollback you committed data to a special time point.", #ifdef WHEN_FLASHBACK_REVIEW_READY "before Flashback feature writing a row, original row can insert to review-dbname.review-tablename," @@ -1656,8 +2104,8 @@ static struct my_option my_options[] = "statements. Output files named after server logs.", &opt_raw_mode, &opt_raw_mode, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"result-file", 'r', "Direct output to a given file. With --raw this is a " - "prefix for the file names.", + {"result-file", 'r', "Direct output to a given file. With --raw or " + "--convert-engine-binlog this is a prefix for the output file names.", &result_file_name, &result_file_name, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, #ifdef WHEN_FLASHBACK_REVIEW_READY @@ -1971,6 +2419,8 @@ static void cleanup() delete_dynamic(&binlog_events); delete_dynamic(&events_in_stmt); } + if (gtid_state) + delete gtid_state; delete engine_binlog_reader; DBUG_VOID_RETURN; } @@ -2554,7 +3004,7 @@ static Exit_status dump_log_entries(const char* logname) Set safe delimiter, to dump things like CREATE PROCEDURE safely */ - if (!opt_raw_mode) + if (!opt_raw_mode && !opt_convert_engine_binlog) fprintf(result_file, "DELIMITER /*!*/;\n"); strmov(print_event_info.delimiter, "/*!*/;"); @@ -2575,7 +3025,8 @@ static Exit_status dump_log_entries(const char* logname) print_event_info.short_form= short_form; print_event_info.print_row_count= print_row_count; print_event_info.file= result_file; - fflush(result_file); + if (result_file) + fflush(result_file); rc= (remote_opt ? dump_remote_log_entries(&print_event_info, logname) : dump_local_log_entries(&print_event_info, logname)); @@ -2585,7 +3036,7 @@ static Exit_status dump_log_entries(const char* logname) return rc; /* Set delimiter back to semicolon */ - if (!opt_raw_mode && !opt_flashback) + if (!opt_raw_mode && !opt_flashback && !opt_convert_engine_binlog) fprintf(result_file, "DELIMITER ;\n"); strmov(print_event_info.delimiter, ";"); return rc; @@ -2826,8 +3277,6 @@ static Exit_status handle_event_text_mode(PRINT_EVENT_INFO *print_event_info, } -static char out_file_name[FN_REFLEN + 1]; - static Exit_status handle_event_raw_mode(PRINT_EVENT_INFO *print_event_info, ulong *len, const char* logname, uint logname_len) @@ -3149,6 +3598,12 @@ static Exit_status check_header(IO_CACHE* file, error("File is not a binary log file."); return ERROR_STOP; } + if (opt_convert_engine_binlog) + { + error("The --convert-engine-binlog option requires InnoDB-engine " + "binlog input files"); + return ERROR_STOP; + } /* Imagine we are running with --start-position=1000. We still need @@ -3369,6 +3824,12 @@ static Exit_status dump_local_log_entries(PRINT_EVENT_INFO *print_event_info, { if (open_engine_binlog(engine_binlog_reader, start_position, logname, file)) goto err; + /* Initialize the GTID state tracker used to generate GTID_LIST_EVENT while + converting the engine binlog to legacy binlog */ + if (opt_convert_engine_binlog && !gtid_state) { + gtid_state= new rpl_binlog_state_base(); + gtid_state->init(); + } } for (;;) { @@ -3444,9 +3905,16 @@ static Exit_status dump_local_log_entries(PRINT_EVENT_INFO *print_event_info, ((ev->get_type_code() == UNKNOWN_EVENT && ((Unknown_log_event *) ev)->what == Unknown_log_event::ENCRYPTED)) || old_off + ev->data_written == my_b_tell(file)); - if ((retval= process_event(print_event_info, ev, old_off, logname)) != - OK_CONTINUE) - goto end; + + if (opt_convert_engine_binlog) { + if ((retval= write_event_to_legacy_binlog(ev)) != OK_CONTINUE) + goto end; + } + else { + if ((retval= process_event(print_event_info, ev, old_off, logname)) != + OK_CONTINUE) + goto end; + } } /* NOTREACHED */ @@ -3455,6 +3923,11 @@ static Exit_status dump_local_log_entries(PRINT_EVENT_INFO *print_event_info, retval= ERROR_STOP; end: + if (output_legacy_binlog_file) + { + my_fclose(output_legacy_binlog_file, MYF(0)); + output_legacy_binlog_file= NULL; + } if (fd >= 0) my_close(fd, MYF(MY_WME)); /* @@ -3516,7 +3989,6 @@ int main(int argc, char** argv) error("The --raw mode is not allowed with --flashback mode"); die(1); } - if (opt_flashback) { my_init_dynamic_array(PSI_NOT_INSTRUMENTED, &binlog_events, @@ -3527,7 +3999,28 @@ int main(int argc, char** argv) if (opt_stop_never) to_last_remote_log= TRUE; - if (opt_raw_mode) + if (opt_convert_engine_binlog) + { + if (remote_opt) + { + error("The --convert-engine-binlog option does not support " + "--read-from-remote-server"); + die(1); + } + if (opt_raw_mode) + { + error("The --convert-engine-binlog option cannot be combined with --raw"); + die(1); + } + if (opt_flashback) + { + error("The --convert-engine-binlog option cannot be combined " + "with --flashback"); + die(1); + } + + } + else if (opt_raw_mode) { if (!remote_opt) { @@ -3583,7 +4076,7 @@ int main(int argc, char** argv) else load_processor.init_by_cur_dir(); - if (!opt_raw_mode) + if (!opt_raw_mode && !opt_convert_engine_binlog) { fprintf(result_file, "/*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/;\n"); @@ -3627,14 +4120,16 @@ int main(int argc, char** argv) if we finished processing input before reaching the stop boundaries indicated by --stop-datetime or --stop-position. */ - if (stop_datetime_given && stop_datetime > last_processed_ev.datetime) + if (!opt_convert_engine_binlog && + stop_datetime_given && stop_datetime > last_processed_ev.datetime) warning("Did not reach stop datetime '%s' before end of input", stop_datetime_str); - if ((static_cast(stop_position) != stop_position_default) && + if (!opt_convert_engine_binlog && + (static_cast(stop_position) != stop_position_default) && stop_position > last_processed_ev.position) warning("Did not reach stop position %llu before end of input", stop_position); - if (position_gtid_filter) + if (!opt_convert_engine_binlog && position_gtid_filter) position_gtid_filter->verify_final_state(); /* @@ -3656,7 +4151,7 @@ int main(int argc, char** argv) } /* Set delimiter back to semicolon */ - if (retval != ERROR_STOP) + if (retval != ERROR_STOP && !opt_convert_engine_binlog) { if (!stop_event_string.is_empty() && result_file) fprintf(result_file, "%s", stop_event_string.ptr()); @@ -3664,7 +4159,7 @@ int main(int argc, char** argv) fprintf(result_file, "DELIMITER ;\n"); } - if (retval != ERROR_STOP && !opt_raw_mode) + if (retval != ERROR_STOP && !opt_raw_mode && !opt_convert_engine_binlog) { /* Issue a ROLLBACK in case the last printed binlog was crashed and had half @@ -3709,7 +4204,7 @@ int main(int argc, char** argv) are processed because it immediately errors (i.e. retval will be ERROR_STOP) */ - if (retval != ERROR_STOP && gtid_state_validator && + if (retval != ERROR_STOP && !opt_convert_engine_binlog && gtid_state_validator && gtid_state_validator->report(stderr, opt_gtid_strict_mode)) retval= ERROR_STOP; diff --git a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.result b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.result new file mode 100644 index 0000000000000..9156e2df665c1 --- /dev/null +++ b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.result @@ -0,0 +1,25 @@ +include/reset_master.inc +set TIMESTAMP= UNIX_TIMESTAMP("1970-01-21 15:32:22"); +*** Generate a small workload into binlog-000000.ibb +CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1, 0), (2, 0), (3, 0); +UPDATE t1 SET b=1 WHERE a=1; +DELETE FROM t1 WHERE a=2; +REPLACE INTO t1 VALUES (3, 3); +SELECT * FROM t1 ORDER BY a; +a b +1 1 +3 3 +FLUSH BINARY LOGS; +*** Convert binlog-000000.ibb to legacy format +*** Converted file contains the synthesized header events +FOUND 1 /Gtid list/ in conv_listing.txt +FOUND 1 /Binlog checkpoint/ in conv_listing.txt +FOUND 1 /Start: binlog v 4/ in conv_listing.txt +*** Round-trip: replay the converted file and compare data +DROP TABLE t1; +SELECT * FROM t1 ORDER BY a; +a b +1 1 +3 3 +DROP TABLE t1; diff --git a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.test b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.test new file mode 100644 index 0000000000000..cfca1082d9ced --- /dev/null +++ b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.test @@ -0,0 +1,61 @@ +# mysqlbinlog_convert.test +# +# Purpose: +# Basic test for MDEV-37605: mariadb-binlog conversion of InnoDB-format +# binlogs (.ibb) into legacy-format binary log +# + +--source include/have_binlog_format_row.inc +--source include/have_innodb_binlog.inc + +--let $datadir= `SELECT @@datadir` + +--source include/reset_master.inc + +# Fixed timestamp for deterministic event listings (with --timezone=GMT-3 +# from the -master.opt file, matching binlog_in_engine.mysqlbinlog). +set TIMESTAMP= UNIX_TIMESTAMP("1970-01-21 15:32:22"); + +--echo *** Generate a small workload into binlog-000000.ibb +CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1, 0), (2, 0), (3, 0); +UPDATE t1 SET b=1 WHERE a=1; +DELETE FROM t1 WHERE a=2; +REPLACE INTO t1 VALUES (3, 3); +SELECT * FROM t1 ORDER BY a; + +# Rotate so binlog-000000.ibb is complete on disk. Waiting for the *next* +# pre-allocated file (binlog-000002.ibb, fully pre-allocated and empty) +# guarantees both 000000 and the new active 000001 exist. +FLUSH BINARY LOGS; +--let $binlog_name= binlog-000002.ibb +--let $binlog_size= 262144 +--source include/wait_for_engine_binlog.inc + +--echo *** Convert binlog-000000.ibb to legacy format +--exec $MYSQL_BINLOG --convert-engine-binlog --result-file=$MYSQL_TMP_DIR/conv $datadir/binlog-000000.ibb + + +--echo *** Converted file contains the synthesized header events +--exec $MYSQL_BINLOG --verbose $MYSQL_TMP_DIR/conv.000001 > $MYSQL_TMP_DIR/conv_listing.txt +--let SEARCH_FILE= $MYSQL_TMP_DIR/conv_listing.txt +--let SEARCH_PATTERN= Gtid list +--source include/search_pattern_in_file.inc +--let SEARCH_PATTERN= Binlog checkpoint +--source include/search_pattern_in_file.inc +--let SEARCH_PATTERN= Start: binlog v 4 +--source include/search_pattern_in_file.inc +--remove_file $MYSQL_TMP_DIR/conv_listing.txt + +--echo *** Round-trip: replay the converted file and compare data +# The replayed GTIDs duplicate the originals, so the replay must not use +# GTID strict mode (same as in binlog_in_engine.mysqlbinlog). +--exec $MYSQL_BINLOG --gtid-strict-mode=0 $MYSQL_TMP_DIR/conv.000001 > $MYSQLTEST_VARDIR/tmp/convert_replay.sql +DROP TABLE t1; +--exec $MYSQL --abort-source-on-error -e "source $MYSQLTEST_VARDIR/tmp/convert_replay.sql;" test +--remove_file $MYSQLTEST_VARDIR/tmp/convert_replay.sql +SELECT * FROM t1 ORDER BY a; + +# Cleanup +--remove_file $MYSQL_TMP_DIR/conv.000001 +DROP TABLE t1; diff --git a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_gtid_list_event.result b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_gtid_list_event.result new file mode 100644 index 0000000000000..b97edd79aecd1 --- /dev/null +++ b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_gtid_list_event.result @@ -0,0 +1,46 @@ +SET @@session.sql_log_bin= 0; +CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB; +CREATE TABLE t2 (a INT PRIMARY KEY) ENGINE=InnoDB; +CREATE TABLE t3 (a INT PRIMARY KEY) ENGINE=InnoDB; +CREATE TABLE t4 (a INT PRIMARY KEY) ENGINE=InnoDB; +SET @@session.sql_log_bin= 1; +include/reset_master.inc +*** Generate GTIDs across several InnoDB binlog files +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +INSERT INTO t1 VALUES (2); +SET @@session.gtid_domain_id= 1; +SET @@session.server_id= 2; +INSERT INTO t2 VALUES (1); +FLUSH BINARY LOGS; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +INSERT INTO t1 VALUES (1); +SET @@session.gtid_domain_id= 2; +SET @@session.server_id= 3; +INSERT INTO t3 VALUES (1); +FLUSH BINARY LOGS; +SET @@session.gtid_domain_id= 1; +SET @@session.server_id= 1; +INSERT INTO t2 VALUES (2); +SET @@session.gtid_domain_id= 3; +SET @@session.server_id= 4; +INSERT INTO t4 VALUES (1); +# restart +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +INSERT INTO t1 VALUES (3); +FLUSH BINARY LOGS; +*** Convert the generated InnoDB binlogs to legacy format +*** Verify synthesized Gtid_list events in converted legacy binlogs +FOUND 1 /Gtid list \[\]/ in gtid_conv_listing.txt +FOUND 1 /Gtid list \[0-1-1,\n# 1-2-1\]/ in gtid_conv_listing.txt +FOUND 1 /Gtid list \[0-1-2,\n# 1-2-1,\n# 2-3-1\]/ in gtid_conv_listing.txt +FOUND 1 /Gtid list \[0-1-2,\n# 1-2-1,\n# 1-1-2,\n# 2-3-1,\n# 3-4-1\]/ in gtid_conv_listing.txt +*** Convert the generated InnoDB binlog(binlog-000002.ibb) to legacy format in random order +*** Verify synthesized Gtid_list events in converted legacy binlogs +NOT FOUND /Gtid list \[0-1-2,\n# 1-2-1,\n# 1-1-2,\n# 2-3-1,\n# 3-4-1\]/ in gtid_conv_listing.txt +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; +DROP TABLE t4; diff --git a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_gtid_list_event.test b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_gtid_list_event.test new file mode 100644 index 0000000000000..d2d8061a2e50d --- /dev/null +++ b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_gtid_list_event.test @@ -0,0 +1,129 @@ +# +# Purpose: +# Verify that mariadb-binlog populates synthesized GTID_LIST_EVENTs correctly +# when converting InnoDB-format binlogs (.ibb) to legacy binlog files. +# +# Methodology: +# Generate several .ibb files with GTIDs across multiple domains and server +# ids, including GTIDs that are not ordered by server id. Restart the server +# in the middle so the converter sees another FORMAT_DESCRIPTION_EVENT and +# rotates the generated legacy output. Convert all completed .ibb files and +# verify each generated legacy binlog contains the expected Gtid_list state. +# Convert the .ibb files again in random order and verify the generated +# legacy binlogs contain the expected Gtid_list state. +# + +--source include/have_binlog_format_row.inc +--source include/have_innodb_binlog.inc + +--let $datadir= `SELECT @@datadir` + +# start of binlog-000000.ibb + +SET @@session.sql_log_bin= 0; +CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB; +CREATE TABLE t2 (a INT PRIMARY KEY) ENGINE=InnoDB; +CREATE TABLE t3 (a INT PRIMARY KEY) ENGINE=InnoDB; +CREATE TABLE t4 (a INT PRIMARY KEY) ENGINE=InnoDB; +SET @@session.sql_log_bin= 1; + +--source include/reset_master.inc + +--echo *** Generate GTIDs across several InnoDB binlog files +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +INSERT INTO t1 VALUES (2); + +SET @@session.gtid_domain_id= 1; +SET @@session.server_id= 2; +INSERT INTO t2 VALUES (1); + +# end of binlog-000000.ibb + +FLUSH BINARY LOGS; + +# start of binlog-000001.ibb + +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +INSERT INTO t1 VALUES (1); + +SET @@session.gtid_domain_id= 2; +SET @@session.server_id= 3; +INSERT INTO t3 VALUES (1); + +# end of binlog-000001.ibb + +FLUSH BINARY LOGS; + +# start of binlog-000002.ibb + +SET @@session.gtid_domain_id= 1; +SET @@session.server_id= 1; +INSERT INTO t2 VALUES (2); + +SET @@session.gtid_domain_id= 3; +SET @@session.server_id= 4; +INSERT INTO t4 VALUES (1); + +--source include/restart_mysqld.inc + +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +INSERT INTO t1 VALUES (3); + +# end of binlog-000002.ibb + +FLUSH BINARY LOGS; + +--let $binlog_name= binlog-000003.ibb +--let $binlog_size= 262144 +--source include/wait_for_engine_binlog.inc + + +--echo *** Convert the generated InnoDB binlogs to legacy format +--exec $MYSQL_BINLOG --convert-engine-binlog --result-file=$MYSQL_TMP_DIR/gtid_conv $datadir/binlog-000000.ibb $datadir/binlog-000001.ibb $datadir/binlog-000002.ibb + +--echo *** Verify synthesized Gtid_list events in converted legacy binlogs +--exec $MYSQL_BINLOG --verbose $MYSQL_TMP_DIR/gtid_conv.000001 > $MYSQL_TMP_DIR/gtid_conv_listing.txt +--let SEARCH_FILE= $MYSQL_TMP_DIR/gtid_conv_listing.txt +--let SEARCH_PATTERN= Gtid list \[\] +--source include/search_pattern_in_file.inc + +--exec $MYSQL_BINLOG --verbose $MYSQL_TMP_DIR/gtid_conv.000002 > $MYSQL_TMP_DIR/gtid_conv_listing.txt +--let SEARCH_PATTERN= Gtid list \[0-1-1,\n# 1-2-1\] +--source include/search_pattern_in_file.inc + +--exec $MYSQL_BINLOG --verbose $MYSQL_TMP_DIR/gtid_conv.000003 > $MYSQL_TMP_DIR/gtid_conv_listing.txt +--let SEARCH_PATTERN= Gtid list \[0-1-2,\n# 1-2-1,\n# 2-3-1\] +--source include/search_pattern_in_file.inc + +--exec $MYSQL_BINLOG --verbose $MYSQL_TMP_DIR/gtid_conv.000004 > $MYSQL_TMP_DIR/gtid_conv_listing.txt +--let SEARCH_PATTERN= Gtid list \[0-1-2,\n# 1-2-1,\n# 1-1-2,\n# 2-3-1,\n# 3-4-1\] +--source include/search_pattern_in_file.inc + +--remove_file $MYSQL_TMP_DIR/gtid_conv_listing.txt +--remove_file $MYSQL_TMP_DIR/gtid_conv.000001 +--remove_file $MYSQL_TMP_DIR/gtid_conv.000002 +--remove_file $MYSQL_TMP_DIR/gtid_conv.000003 +--remove_file $MYSQL_TMP_DIR/gtid_conv.000004 + + +--echo *** Convert the generated InnoDB binlog(binlog-000002.ibb) to legacy format in random order +--exec $MYSQL_BINLOG --convert-engine-binlog --result-file=$MYSQL_TMP_DIR/gtid_conv $datadir/binlog-000002.ibb + +--echo *** Verify synthesized Gtid_list events in converted legacy binlogs +--exec $MYSQL_BINLOG --verbose $MYSQL_TMP_DIR/gtid_conv.000001 > $MYSQL_TMP_DIR/gtid_conv_listing.txt +--let SEARCH_FILE= $MYSQL_TMP_DIR/gtid_conv_listing.txt +--let SEARCH_PATTERN= Gtid list \[0-1-2,\n# 1-2-1,\n# 1-1-2,\n# 2-3-1,\n# 3-4-1\] +# TODO: Tarun this search will fail now as this functionality is not yet implemented +--source include/search_pattern_in_file.inc + +--remove_file $MYSQL_TMP_DIR/gtid_conv_listing.txt +--remove_file $MYSQL_TMP_DIR/gtid_conv.000001 + + +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; +DROP TABLE t4; diff --git a/sql/log_event.cc b/sql/log_event.cc index 8e6de277a080e..3e29d057a107d 100644 --- a/sql/log_event.cc +++ b/sql/log_event.cc @@ -2370,6 +2370,60 @@ Format_description_log_event(const uchar *buf, uint event_len, DBUG_VOID_RETURN; } +bool +Format_description_log_event::to_packet(String *packet) +{ + uchar *p; + uint32 needed_length= + packet->length() + START_V3_HEADER_LEN + 1 + number_of_event_types + 1; + if (packet->reserve(needed_length)) + return true; + p= (uchar *)packet->ptr() + packet->length(); + packet->length(needed_length); + int2store(p, binlog_version); + p += 2; + memcpy(p, server_version, ST_SERVER_VER_LEN); + p+= ST_SERVER_VER_LEN; +/* TODO: Tarun get this reviewed */ +#ifdef MYSQL_SERVER + if (!dont_set_created) + created= get_time(); +#endif + int4store(p, created); + p+= 4; + *p++= common_header_len; + memcpy(p, post_header_len, number_of_event_types); + p+= number_of_event_types; + + /* + if checksum is requested + record the checksum-algorithm descriptor next to + post_header_len vector which will be followed by the checksum value. + Master is supposed to trigger checksum computing by binlog_checksum_options, + slave does it via marking the event according to + FD_queue checksum_alg value. + */ + compile_time_assert(BINLOG_CHECKSUM_ALG_DESC_LEN == 1); + uint8 checksum_byte= (uint8) (used_checksum_alg != BINLOG_CHECKSUM_ALG_UNDEF ? + used_checksum_alg : BINLOG_CHECKSUM_ALG_OFF); + DBUG_ASSERT(used_checksum_alg != BINLOG_CHECKSUM_ALG_UNDEF); + /* + FD of checksum-aware server is always checksum-equipped, (V) is in, + regardless of @@global.binlog_checksum policy. + Thereby a combination of (A) == 0, (V) != 0 means + it's the checksum-aware server's FD event that heads checksum-free binlog + file. + Here 0 stands for checksumming OFF to evaluate (V) as 0 is that case. + A combination of (A) != 0, (V) != 0 denotes FD of the checksum-aware server + heading the checksummed binlog. + (A), (V) presence in FD of the checksum-aware server makes the event + 1 + 4 bytes bigger comparing to the former FD. + */ + *p++= checksum_byte; + + return false; +} + bool Format_description_log_event::start_decryption(Start_encryption_log_event* sele) { DBUG_ASSERT(crypto_data.scheme == 0); @@ -2560,6 +2614,16 @@ Rotate_log_event::Rotate_log_event(const uchar *buf, uint event_len, Binlog_checkpoint_log_event methods **************************************************************************/ +Binlog_checkpoint_log_event::Binlog_checkpoint_log_event( + const char *binlog_file_name_arg, uint binlog_file_len_arg) + : Log_event(), + binlog_file_name(my_strndup(PSI_INSTRUMENT_ME, binlog_file_name_arg, + binlog_file_len_arg, MYF(MY_WME))), + binlog_file_len(binlog_file_len_arg) +{ + cache_type= EVENT_NO_CACHE; +} + Binlog_checkpoint_log_event::Binlog_checkpoint_log_event( const uchar *buf, uint event_len, const Format_description_log_event *description_event) @@ -2771,6 +2835,35 @@ Gtid_list_log_event::Gtid_list_log_event(const uchar *buf, uint event_len, #endif } +bool +Gtid_list_log_event::to_packet(String *packet) +{ + uint32 i; + uchar *p; + uint32 needed_length; + + DBUG_ASSERT(count < 1<<28); + + needed_length= packet->length() + get_data_size(); + if (packet->reserve(needed_length)) + return true; + p= (uchar *)packet->ptr() + packet->length();; + packet->length(needed_length); + int4store(p, (count & ((1<<28)-1)) | gl_flags); + p += 4; + /* Initialise the padding for empty Gtid_list. */ + if (count == 0) + int2store(p, 0); + for (i= 0; i < count; ++i) + { + int4store(p, list[i].domain_id); + int4store(p+4, list[i].server_id); + int8store(p+8, list[i].seq_no); + p += 16; + } + + return false; +} /* Used to record gtid_list event while sending binlog to slave, without having to diff --git a/sql/log_event.h b/sql/log_event.h index 313a81c879c70..f2edc379334f1 100644 --- a/sql/log_event.h +++ b/sql/log_event.h @@ -2662,10 +2662,11 @@ class Format_description_log_event: public Log_event #ifdef HAVE_REPLICATION void pack_info(Protocol* protocol) override; #endif /* HAVE_REPLICATION */ - bool to_packet(String *packet); #else bool print(FILE* file, PRINT_EVENT_INFO* print_event_info) override; #endif + bool to_packet(String *packet); + bool header_is_valid() const { return common_header_len >= LOG_EVENT_MINIMAL_HEADER_LEN && post_header_len; @@ -3329,14 +3330,15 @@ class Binlog_checkpoint_log_event: public Log_event uint binlog_file_len; #ifdef MYSQL_SERVER - Binlog_checkpoint_log_event(const char *binlog_file_name_arg, - uint binlog_file_len_arg); + #ifdef HAVE_REPLICATION void pack_info(Protocol *protocol) override; #endif #else bool print(FILE *file, PRINT_EVENT_INFO *print_event_info) override; #endif + Binlog_checkpoint_log_event(const char *binlog_file_name_arg, + uint binlog_file_len_arg); Binlog_checkpoint_log_event(const uchar *buf, uint event_len, const Format_description_log_event *description_event); @@ -3679,6 +3681,7 @@ class Gtid_list_log_event: public Log_event void pack_info(Protocol *protocol) override; #endif #else + Gtid_list_log_event(rpl_binlog_state_base *gtid_set); bool print(FILE *file, PRINT_EVENT_INFO *print_event_info) override; #endif Gtid_list_log_event(const uchar *buf, uint event_len, @@ -3694,8 +3697,8 @@ class Gtid_list_log_event: public Log_event GTID_LIST_HEADER_LEN+2 : GTID_LIST_HEADER_LEN+count*element_size); } bool is_valid() const override { return list != NULL; } -#if defined(MYSQL_SERVER) && defined(HAVE_REPLICATION) bool to_packet(String *packet); +#if defined(MYSQL_SERVER) && defined(HAVE_REPLICATION) bool write(Log_event_writer *writer) override; int do_apply_event(rpl_group_info *rgi) override; enum_skip_reason do_shall_skip(rpl_group_info *rgi) override; diff --git a/sql/log_event_client.cc b/sql/log_event_client.cc index fd1648fe92502..5927730c0ad45 100644 --- a/sql/log_event_client.cc +++ b/sql/log_event_client.cc @@ -2392,6 +2392,24 @@ bool Binlog_checkpoint_log_event::print(FILE *file, return cache.flush_data(); } +/* + + Constructor for Gtid_list_log_event. + Used in mysqlbinlog to generate GTID_LIST_EVENT while converting the engine binlog to legacy binlog. + TODO: Tarun get this reviewed. +*/ +Gtid_list_log_event::Gtid_list_log_event(rpl_binlog_state_base *gtid_set) + : count(gtid_set->count_nolock()), gl_flags(0), list(0), sub_id_list(0) +{ + cache_type= EVENT_NO_CACHE; + /* Failure to allocate memory will be caught by is_valid() returning false. */ + if (count < (1<<28) && + (list = (rpl_gtid *)my_malloc(PSI_INSTRUMENT_ME, + count * sizeof(*list) + (count == 0), MYF(MY_WME)))) + { + gtid_set->get_gtid_list_nolock(list, count); + } +} bool Gtid_list_log_event::print(FILE *file, PRINT_EVENT_INFO *print_event_info) diff --git a/sql/log_event_server.cc b/sql/log_event_server.cc index 79c5b3bdf1c11..7424e4794d17c 100644 --- a/sql/log_event_server.cc +++ b/sql/log_event_server.cc @@ -2454,56 +2454,7 @@ void Format_description_log_event::pack_info(Protocol *protocol) } #endif /* defined(HAVE_REPLICATION) */ -bool -Format_description_log_event::to_packet(String *packet) -{ - uchar *p; - uint32 needed_length= - packet->length() + START_V3_HEADER_LEN + 1 + number_of_event_types + 1; - if (packet->reserve(needed_length)) - return true; - p= (uchar *)packet->ptr() + packet->length();; - packet->length(needed_length); - int2store(p, binlog_version); - p += 2; - memcpy(p, server_version, ST_SERVER_VER_LEN); - p+= ST_SERVER_VER_LEN; - if (!dont_set_created) - created= get_time(); - int4store(p, created); - p+= 4; - *p++= common_header_len; - memcpy(p, post_header_len, number_of_event_types); - p+= number_of_event_types; - - /* - if checksum is requested - record the checksum-algorithm descriptor next to - post_header_len vector which will be followed by the checksum value. - Master is supposed to trigger checksum computing by binlog_checksum_options, - slave does it via marking the event according to - FD_queue checksum_alg value. - */ - compile_time_assert(BINLOG_CHECKSUM_ALG_DESC_LEN == 1); - uint8 checksum_byte= (uint8) (used_checksum_alg != BINLOG_CHECKSUM_ALG_UNDEF ? - used_checksum_alg : BINLOG_CHECKSUM_ALG_OFF); - DBUG_ASSERT(used_checksum_alg != BINLOG_CHECKSUM_ALG_UNDEF); - /* - FD of checksum-aware server is always checksum-equipped, (V) is in, - regardless of @@global.binlog_checksum policy. - Thereby a combination of (A) == 0, (V) != 0 means - it's the checksum-aware server's FD event that heads checksum-free binlog - file. - Here 0 stands for checksumming OFF to evaluate (V) as 0 is that case. - A combination of (A) != 0, (V) != 0 denotes FD of the checksum-aware server - heading the checksummed binlog. - (A), (V) presence in FD of the checksum-aware server makes the event - 1 + 4 bytes bigger comparing to the former FD. - */ - *p++= checksum_byte; - return false; -} bool Format_description_log_event::write(Log_event_writer *writer) { @@ -2879,17 +2830,6 @@ Binlog_checkpoint_log_event::do_shall_skip(rpl_group_info *rgi) #endif -Binlog_checkpoint_log_event::Binlog_checkpoint_log_event( - const char *binlog_file_name_arg, - uint binlog_file_len_arg) - :Log_event(), - binlog_file_name(my_strndup(PSI_INSTRUMENT_ME, binlog_file_name_arg, binlog_file_len_arg, - MYF(MY_WME))), - binlog_file_len(binlog_file_len_arg) -{ - cache_type= EVENT_NO_CACHE; -} - bool Binlog_checkpoint_log_event::write(Log_event_writer *writer) { @@ -3449,36 +3389,6 @@ Gtid_list_log_event::Gtid_list_log_event(slave_connection_state *gtid_set, #if defined(HAVE_REPLICATION) -bool -Gtid_list_log_event::to_packet(String *packet) -{ - uint32 i; - uchar *p; - uint32 needed_length; - - DBUG_ASSERT(count < 1<<28); - - needed_length= packet->length() + get_data_size(); - if (packet->reserve(needed_length)) - return true; - p= (uchar *)packet->ptr() + packet->length();; - packet->length(needed_length); - int4store(p, (count & ((1<<28)-1)) | gl_flags); - p += 4; - /* Initialise the padding for empty Gtid_list. */ - if (count == 0) - int2store(p, 0); - for (i= 0; i < count; ++i) - { - int4store(p, list[i].domain_id); - int4store(p+4, list[i].server_id); - int8store(p+8, list[i].seq_no); - p += 16; - } - - return false; -} - bool Gtid_list_log_event::write(Log_event_writer *writer) diff --git a/sql/rpl_gtid.cc b/sql/rpl_gtid.cc index 430e4a4bfc64a..b40ebeafcbf7a 100644 --- a/sql/rpl_gtid.cc +++ b/sql/rpl_gtid.cc @@ -1517,6 +1517,7 @@ rpl_slave_state::alloc_gtid_pos_table(LEX_CSTRING *table_name, void *hton, return p; } +#endif void rpl_binlog_state_base::init() @@ -1694,6 +1695,7 @@ rpl_binlog_state_base::find_nolock(uint32 domain_id, uint32 server_id) sizeof(server_id)); } +#ifndef MYSQL_CLIENT /* Return true if this binlog state is before the position specified by the @@ -1895,10 +1897,12 @@ rpl_binlog_state::update_with_next_gtid(uint32 domain_id, uint32 server_id, return res; } - +#endif // MYSQL_CLIENT +/* TODO: Tarun get this reviewed. changed the base class name from +rpl_gtid_base to rpl_binlog_state_base (also get reviewed the ifndef blocks )*/ /* Helper functions for update. */ int -rpl_binlog_state::element::update_element(const rpl_gtid *gtid) +rpl_binlog_state_base::element::update_element(const rpl_gtid *gtid) { rpl_gtid *lookup_gtid; @@ -1938,6 +1942,7 @@ rpl_binlog_state::element::update_element(const rpl_gtid *gtid) return 0; } +#ifndef MYSQL_CLIENT /* Check that a new GTID can be logged without creating an out-of-order From 3b7263bc05424a1db5e9f13b6b7cbab61a256401 Mon Sep 17 00:00:00 2001 From: Tarun Wadhwa Date: Mon, 29 Jun 2026 21:56:34 +0000 Subject: [PATCH 02/14] Added error handling/message for log_file_pos (end_log_pos) --- client/mysqlbinlog.cc | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/client/mysqlbinlog.cc b/client/mysqlbinlog.cc index 3de59f34bc0dc..802a544a0f0b6 100644 --- a/client/mysqlbinlog.cc +++ b/client/mysqlbinlog.cc @@ -1574,6 +1574,18 @@ Exit_status process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev, ############################################################################## */ +static bool store_log_file_pos(uchar *pos) +{ + if (log_file_pos > UINT_MAX32) + { + error("Converted binlog output file exceeds the maximum supported size of " + "4GB"); + return true; + } + int4store(pos, log_file_pos); + return false; +} + static bool write_event_header(FILE *outfile, Log_event_type event_type, ulong extra_len, time_t timestamp, my_bool *do_checksum, ha_checksum *crc, @@ -1604,7 +1616,9 @@ static bool write_event_header(FILE *outfile, Log_event_type event_type, int2store(header + FLAGS_OFFSET, 0); /* Update the log_file_pos */ log_file_pos+= event_len; - int4store(header + LOG_POS_OFFSET, log_file_pos); + if (store_log_file_pos(header + LOG_POS_OFFSET)) + return true; + /* Write this header to outfile */ if (my_fwrite(outfile, (const uchar *)header, LOG_EVENT_HEADER_LEN, MYF(MY_NABP))) { error("Could not write header into converted binlog file '%s'", @@ -1641,11 +1655,14 @@ write_event_footer(FILE *outfile, my_bool do_checksum, ha_checksum crc) It does not update the log_pos variable of ev as it is not really needed for the conversion. */ -static void update_event_end_log_pos(Log_event *ev) +static bool update_event_end_log_pos(Log_event *ev) { - DBUG_ASSERT(ev->temp_buf != NULL && ev->data_written >= LOG_EVENT_HEADER_LEN); + DBUG_ASSERT(ev->temp_buf != NULL && + ev->data_written >= LOG_EVENT_HEADER_LEN); log_file_pos+= ev->data_written; - int4store(&ev->temp_buf[LOG_POS_OFFSET], log_file_pos); + if (store_log_file_pos(ev->temp_buf + LOG_POS_OFFSET)) + return true; + return false; } /* @@ -1667,7 +1684,8 @@ static bool write_format_description_event_to_legacy_binlog( // temp_buf stores the raw bytes of the event and data_written is the length of those raw bytes if(fdev->temp_buf) { /* Update the log_file_pos */ - update_event_end_log_pos(fdev); + if (update_event_end_log_pos(fdev)) + return true; /* recompute checksum */ update_checksum(fdev); @@ -1980,7 +1998,8 @@ static Exit_status write_event_to_legacy_binlog(Log_event *ev) } /* Update the log_file_pos */ - update_event_end_log_pos(ev); + if (update_event_end_log_pos(ev)) + goto err; // ev->temp_buf contains the raw event bytes and ev->data_written is the length of the event if (my_fwrite(output_legacy_binlog_file, (const uchar *)ev->temp_buf, @@ -3925,6 +3944,7 @@ static Exit_status dump_local_log_entries(PRINT_EVENT_INFO *print_event_info, end: if (output_legacy_binlog_file) { + /* TODO: Tarun write the STOP_EVENT maybe? */ my_fclose(output_legacy_binlog_file, MYF(0)); output_legacy_binlog_file= NULL; } From 0af9fe9556960ec2d70ffad4784ebbf6b37b9fd6 Mon Sep 17 00:00:00 2001 From: Tarun Wadhwa Date: Wed, 1 Jul 2026 23:20:11 +0000 Subject: [PATCH 03/14] Added serializer for ROTATE_EVENT. And the converter should append the ROTATE_EVENT whenever a binlog file is rotated. --- client/mysqlbinlog.cc | 114 +++++++++++++++--- ...lbinlog_convert_engine_binlog_basic.result | 1 + ...sqlbinlog_convert_engine_binlog_basic.test | 3 + ...nvert_engine_binlog_gtid_list_event.result | 2 + ...convert_engine_binlog_gtid_list_event.test | 18 ++- sql/sql_repl.cc | 1 + 6 files changed, 115 insertions(+), 24 deletions(-) diff --git a/client/mysqlbinlog.cc b/client/mysqlbinlog.cc index 802a544a0f0b6..713d199d70083 100644 --- a/client/mysqlbinlog.cc +++ b/client/mysqlbinlog.cc @@ -1812,6 +1812,12 @@ static bool write_gtid_list_event_to_legacy_binlog(FILE *outfile, return false; } +/* + TODO: Tarun get this reviewed. should i pass the event to this function + or alternatively i can just pass the required args + (binlog_file_name, binlog_file_len) to this function + and we don't have to extract the constructor for BINLOG_CHECKPOINT_EVENT +*/ static bool write_binlog_checkpoint_event_to_legacy_binlog( FILE *outfile, Binlog_checkpoint_log_event *bcle) { @@ -1876,6 +1882,64 @@ static bool write_binlog_checkpoint_event_to_legacy_binlog( return false; } +static bool +write_rotate_log_event_to_legacy_binlog(FILE *outfile, + const char *binlog_file_name) +{ + char buf[ROTATE_HEADER_LEN]; + my_bool do_checksum; + ha_checksum crc= 0; + + const char *p= binlog_file_name + dirname_length(binlog_file_name); + uint ident_len= (uint) strlen(p); + + /* Write header of ROTATE_EVENT */ + /* TODO: Tarun handle the timestamp */ + time_t ts= 0; + if (write_event_header(outfile, ROTATE_EVENT, ident_len + ROTATE_HEADER_LEN, + ts, &do_checksum, &crc, BINLOG_CHECKSUM_ALG_OFF)) + { + error("Could not write ROTATE_EVENT header to output legacy binlog file"); + return true; + } + + /* Write body of ROTATE_EVENT */ + int8store(buf + R_POS_OFFSET, BIN_LOG_HEADER_SIZE); + + if (my_fwrite(outfile, (const uchar *) buf, ROTATE_HEADER_LEN, MYF(MY_NABP))) + { + error("Could not write body into converted binlog file '%s'", + out_file_name); + return true; + } + + if (my_fwrite(outfile, (const uchar *) p, ident_len, MYF(MY_NABP))) + { + error("Could not write body into converted binlog file '%s'", + out_file_name); + return true; + } + + /* we are not writing the footer because we are not supporting the checksum + in every event */ + + return false; +} + + +/* + Generates the output legacy binlog file name based on the prefix and the index + @param out_name: Char buffer to store the result + @param out_name_len: Size of the char buffer + @param index: Index of the output legacy binlog file +*/ +static void generate_output_legacy_binlog_name(char *out_name, size_t out_name_len, ulonglong index) +{ + const char *prefix= result_file_name ? result_file_name + : default_output_legacy_binlog_prefix; + my_snprintf(out_name, out_name_len, "%s.%06llu", prefix, index); +} + static bool init_output_legacy_binlog(FILE **out_file, char *out_name, size_t out_name_len) { @@ -1883,11 +1947,7 @@ static bool init_output_legacy_binlog(FILE **out_file, char *out_name, /* Reset the log_file_pos to 0 for the new output legacy binlog file */ log_file_pos= 0; - const char *prefix= result_file_name ? result_file_name - : default_output_legacy_binlog_prefix; - - my_snprintf(out_name, out_name_len, "%s.%06llu", prefix, - ++convert_engine_output_index); + generate_output_legacy_binlog_name(out_name, out_name_len, ++convert_engine_output_index); if (!(*out_file= my_fopen(out_name, O_WRONLY | O_BINARY, MYF(MY_WME)))) { @@ -1953,15 +2013,27 @@ static bool init_output_legacy_binlog(FILE **out_file, char *out_name, static Exit_status write_event_to_legacy_binlog(Log_event *ev) { - // if event type is FORMAT_DESCRIPTION_EVENT, store the event in global variable glob_description_event - if (ev->get_type_code() == FORMAT_DESCRIPTION_EVENT) { + // if event type is FORMAT_DESCRIPTION_EVENT, store the event in global + // variable glob_description_event + if (ev->get_type_code() == FORMAT_DESCRIPTION_EVENT) + { delete glob_description_event; - glob_description_event= (Format_description_log_event*) ev; + glob_description_event= (Format_description_log_event *) ev; // close the output legacy binlog file if it is open - if (output_legacy_binlog_file) { - // TODO: tarun Write the ROTATE_EVENT to the output legacy binlog file + if (output_legacy_binlog_file) + { + char next_out_file_name[FN_REFLEN + 1]; + generate_output_legacy_binlog_name(next_out_file_name, + sizeof(next_out_file_name), + convert_engine_output_index + 1); + + /* Write the ROTATE_EVENT to the output legacy binlog file */ + if (write_rotate_log_event_to_legacy_binlog(output_legacy_binlog_file, + next_out_file_name)) + goto err; + my_fclose(output_legacy_binlog_file, MYF(0)); output_legacy_binlog_file= NULL; } @@ -1971,7 +2043,8 @@ static Exit_status write_event_to_legacy_binlog(Log_event *ev) /* Open and initialize the output legacy binlog file. - Writes the FORMAT_DESCRIPTION_EVENT, GTID_LIST event, and BINLOG_CHECKPOINT event. + Writes the FORMAT_DESCRIPTION_EVENT, GTID_LIST event, and BINLOG_CHECKPOINT + event. */ if (!output_legacy_binlog_file) { @@ -1983,15 +2056,17 @@ static Exit_status write_event_to_legacy_binlog(Log_event *ev) /* if event type is GTID_EVENT, update the gtid_state - which will be used to write the GTID_LIST_EVENT in + which will be used to write the GTID_LIST_EVENT in the next output legacy binlog file. */ - if (ev->get_type_code() == GTID_EVENT) { + if (ev->get_type_code() == GTID_EVENT) + { rpl_gtid ev_gtid; - Gtid_log_event *gle= (Gtid_log_event*) ev; + Gtid_log_event *gle= (Gtid_log_event *) ev; ev_gtid= {gle->domain_id, gle->server_id, gle->seq_no}; - if (gtid_state->update_nolock(&ev_gtid)) { + if (gtid_state->update_nolock(&ev_gtid)) + { error("Failed to update GTID state"); goto err; } @@ -2001,12 +2076,12 @@ static Exit_status write_event_to_legacy_binlog(Log_event *ev) if (update_event_end_log_pos(ev)) goto err; - // ev->temp_buf contains the raw event bytes and ev->data_written is the length of the event - if (my_fwrite(output_legacy_binlog_file, (const uchar *)ev->temp_buf, + // ev->temp_buf contains the raw event bytes and ev->data_written is the + // length of the event + if (my_fwrite(output_legacy_binlog_file, (const uchar *) ev->temp_buf, ev->data_written, MYF(MY_NABP))) { - error("Could not write into converted binlog file '%s'", - out_file_name); + error("Could not write into converted binlog file '%s'", out_file_name); goto err; } fflush(output_legacy_binlog_file); @@ -2019,7 +2094,6 @@ static Exit_status write_event_to_legacy_binlog(Log_event *ev) return ERROR_STOP; } - static struct my_option my_options[] = { {"help", '?', "Display this help and exit.", diff --git a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.result b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.result index 9156e2df665c1..74fa34cc64c52 100644 --- a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.result +++ b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.result @@ -16,6 +16,7 @@ FLUSH BINARY LOGS; FOUND 1 /Gtid list/ in conv_listing.txt FOUND 1 /Binlog checkpoint/ in conv_listing.txt FOUND 1 /Start: binlog v 4/ in conv_listing.txt +NOT FOUND /Rotate to/ in conv_listing.txt *** Round-trip: replay the converted file and compare data DROP TABLE t1; SELECT * FROM t1 ORDER BY a; diff --git a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.test b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.test index cfca1082d9ced..68d963da97545 100644 --- a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.test +++ b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.test @@ -45,6 +45,9 @@ FLUSH BINARY LOGS; --source include/search_pattern_in_file.inc --let SEARCH_PATTERN= Start: binlog v 4 --source include/search_pattern_in_file.inc +# it should not contain the Rotate event in the converted file +--let SEARCH_PATTERN= Rotate to +--source include/search_pattern_in_file.inc --remove_file $MYSQL_TMP_DIR/conv_listing.txt --echo *** Round-trip: replay the converted file and compare data diff --git a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_gtid_list_event.result b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_gtid_list_event.result index b97edd79aecd1..f31206ace645f 100644 --- a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_gtid_list_event.result +++ b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_gtid_list_event.result @@ -36,10 +36,12 @@ FLUSH BINARY LOGS; FOUND 1 /Gtid list \[\]/ in gtid_conv_listing.txt FOUND 1 /Gtid list \[0-1-1,\n# 1-2-1\]/ in gtid_conv_listing.txt FOUND 1 /Gtid list \[0-1-2,\n# 1-2-1,\n# 2-3-1\]/ in gtid_conv_listing.txt +FOUND 1 /Rotate to gtid_conv.000004/ in gtid_conv_listing.txt FOUND 1 /Gtid list \[0-1-2,\n# 1-2-1,\n# 1-1-2,\n# 2-3-1,\n# 3-4-1\]/ in gtid_conv_listing.txt *** Convert the generated InnoDB binlog(binlog-000002.ibb) to legacy format in random order *** Verify synthesized Gtid_list events in converted legacy binlogs NOT FOUND /Gtid list \[0-1-2,\n# 1-2-1,\n# 1-1-2,\n# 2-3-1,\n# 3-4-1\]/ in gtid_conv_listing.txt +FOUND 1 /Rotate to gtid_conv.000002/ in gtid_conv_listing.txt DROP TABLE t1; DROP TABLE t2; DROP TABLE t3; diff --git a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_gtid_list_event.test b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_gtid_list_event.test index d2d8061a2e50d..721a37318e0e2 100644 --- a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_gtid_list_event.test +++ b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_gtid_list_event.test @@ -2,6 +2,7 @@ # Purpose: # Verify that mariadb-binlog populates synthesized GTID_LIST_EVENTs correctly # when converting InnoDB-format binlogs (.ibb) to legacy binlog files. +# Also verify that the generated legacy binlogs contain the expected ROTATE_EVENT. # # Methodology: # Generate several .ibb files with GTIDs across multiple domains and server @@ -38,7 +39,7 @@ SET @@session.gtid_domain_id= 1; SET @@session.server_id= 2; INSERT INTO t2 VALUES (1); -# end of binlog-000000.ibb +# end of binlog-000000.ibb which will be converted to gtid_conv.000001 FLUSH BINARY LOGS; @@ -52,7 +53,7 @@ SET @@session.gtid_domain_id= 2; SET @@session.server_id= 3; INSERT INTO t3 VALUES (1); -# end of binlog-000001.ibb +# end of binlog-000001.ibb which will be converted to gtid_conv.000002 FLUSH BINARY LOGS; @@ -66,14 +67,17 @@ SET @@session.gtid_domain_id= 3; SET @@session.server_id= 4; INSERT INTO t4 VALUES (1); +# this will trigger a ROTATE_EVENT and will be converted to/end of gtid_conv.000003 + --source include/restart_mysqld.inc +# start of converted gtid_conv.000004 + SET @@session.gtid_domain_id= 0; SET @@session.server_id= 1; INSERT INTO t1 VALUES (3); -# end of binlog-000002.ibb - +# end of binlog-000002.ibb which will be converted to gtid_conv.000004 FLUSH BINARY LOGS; --let $binlog_name= binlog-000003.ibb @@ -98,6 +102,9 @@ FLUSH BINARY LOGS; --let SEARCH_PATTERN= Gtid list \[0-1-2,\n# 1-2-1,\n# 2-3-1\] --source include/search_pattern_in_file.inc +--let SEARCH_PATTERN= Rotate to gtid_conv.000004 +--source include/search_pattern_in_file.inc + --exec $MYSQL_BINLOG --verbose $MYSQL_TMP_DIR/gtid_conv.000004 > $MYSQL_TMP_DIR/gtid_conv_listing.txt --let SEARCH_PATTERN= Gtid list \[0-1-2,\n# 1-2-1,\n# 1-1-2,\n# 2-3-1,\n# 3-4-1\] --source include/search_pattern_in_file.inc @@ -119,6 +126,9 @@ FLUSH BINARY LOGS; # TODO: Tarun this search will fail now as this functionality is not yet implemented --source include/search_pattern_in_file.inc +--let SEARCH_PATTERN= Rotate to gtid_conv.000002 +--source include/search_pattern_in_file.inc + --remove_file $MYSQL_TMP_DIR/gtid_conv_listing.txt --remove_file $MYSQL_TMP_DIR/gtid_conv.000001 diff --git a/sql/sql_repl.cc b/sql/sql_repl.cc index e178ca785026f..4e80e939adc48 100644 --- a/sql/sql_repl.cc +++ b/sql/sql_repl.cc @@ -225,6 +225,7 @@ static int fake_rotate_event(binlog_send_info *info, ulonglong position, { DBUG_ENTER("fake_rotate_event"); ulong ev_offset; + /* TODO: Tarun get this reviewed. buf should be buf[ROTATE_HEADER_LEN] */ char buf[ROTATE_HEADER_LEN+100]; my_bool do_checksum; int err; From 42fbbffa0e6040ff8574286e8247f75a5b802ee9 Mon Sep 17 00:00:00 2001 From: Tarun Wadhwa Date: Tue, 14 Jul 2026 12:06:37 +0000 Subject: [PATCH 04/14] Implemented support for max-binlog-size flag. It rotates the binlog on GTID_EVENT whenever the max binlog size is reached --- client/mysqlbinlog.cc | 48 ++++++++++++++----- mysql-test/include/have_innodb_binlog.inc | 1 + ...sqlbinlog_convert_engine_binlog_basic.test | 1 - ...nlog_convert_engine_binlog_max_size.result | 15 ++++++ ...binlog_convert_engine_binlog_max_size.test | 40 ++++++++++++++++ 5 files changed, 92 insertions(+), 13 deletions(-) create mode 100644 mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_max_size.result create mode 100644 mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_max_size.test diff --git a/client/mysqlbinlog.cc b/client/mysqlbinlog.cc index 713d199d70083..c5f13c2cf94fb 100644 --- a/client/mysqlbinlog.cc +++ b/client/mysqlbinlog.cc @@ -129,6 +129,7 @@ static my_bool debug_info_flag, debug_check_flag; static my_bool force_if_open_opt= 1; static my_bool opt_raw_mode= 0, opt_stop_never= 0; static my_bool opt_convert_engine_binlog= 0; +static ulong opt_max_binlog_size= 1024L * 1024L * 1024L; my_bool opt_gtid_strict_mode= true; static ulong opt_stop_never_slave_server_id= 0; static my_bool opt_verify_binlog_checksum= 1; @@ -1943,7 +1944,6 @@ static void generate_output_legacy_binlog_name(char *out_name, size_t out_name_l static bool init_output_legacy_binlog(FILE **out_file, char *out_name, size_t out_name_len) { - /* Reset the log_file_pos to 0 for the new output legacy binlog file */ log_file_pos= 0; @@ -2005,6 +2005,23 @@ static bool init_output_legacy_binlog(FILE **out_file, char *out_name, return false; } +static bool rotate_output_legacy_binlog(FILE **out_file, char *out_name, + size_t out_name_len) +{ + char next_out_file_name[FN_REFLEN + 1]; + generate_output_legacy_binlog_name(next_out_file_name, + sizeof(next_out_file_name), + convert_engine_output_index + 1); + + if (write_rotate_log_event_to_legacy_binlog(*out_file, next_out_file_name)) + return true; + + my_fclose(*out_file, MYF(0)); + *out_file= NULL; + + return init_output_legacy_binlog(out_file, out_name, out_name_len); +} + /* Writes the event to the converted legacy binlog file @param ev: The event to write @@ -2024,18 +2041,9 @@ static Exit_status write_event_to_legacy_binlog(Log_event *ev) // close the output legacy binlog file if it is open if (output_legacy_binlog_file) { - char next_out_file_name[FN_REFLEN + 1]; - generate_output_legacy_binlog_name(next_out_file_name, - sizeof(next_out_file_name), - convert_engine_output_index + 1); - - /* Write the ROTATE_EVENT to the output legacy binlog file */ - if (write_rotate_log_event_to_legacy_binlog(output_legacy_binlog_file, - next_out_file_name)) + if (rotate_output_legacy_binlog(&output_legacy_binlog_file, + out_file_name, sizeof(out_file_name))) goto err; - - my_fclose(output_legacy_binlog_file, MYF(0)); - output_legacy_binlog_file= NULL; } return OK_CONTINUE; @@ -2054,6 +2062,17 @@ static Exit_status write_event_to_legacy_binlog(Log_event *ev) goto err; } + /* + Rotate the output legacy binlog file once the max binlog size is reached. + */ + if (ev->get_type_code() == GTID_EVENT && + log_file_pos >= opt_max_binlog_size) + { + if (rotate_output_legacy_binlog(&output_legacy_binlog_file, out_file_name, + sizeof(out_file_name))) + goto err; + } + /* if event type is GTID_EVENT, update the gtid_state which will be used to write the GTID_LIST_EVENT in @@ -2150,6 +2169,11 @@ static struct my_option my_options[] = "Convert InnoDB based engine binlog files to legacy binlog files.", &opt_convert_engine_binlog, &opt_convert_engine_binlog, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + /* TODO: Tarun what should be the max value? */ + {"max-binlog-size", 0, + "Maximum size of converted legacy binlog files.", + &opt_max_binlog_size, &opt_max_binlog_size, 0, GET_ULONG, REQUIRED_ARG, + 1024L * 1024L * 1024L, IO_SIZE, 1024L * 1024L * 1024L, 0, IO_SIZE, 0}, {"flashback", 'B', "Flashback feature can rollback you committed data to a special time point.", #ifdef WHEN_FLASHBACK_REVIEW_READY "before Flashback feature writing a row, original row can insert to review-dbname.review-tablename," diff --git a/mysql-test/include/have_innodb_binlog.inc b/mysql-test/include/have_innodb_binlog.inc index c841fece70297..34761ad865ca6 100644 --- a/mysql-test/include/have_innodb_binlog.inc +++ b/mysql-test/include/have_innodb_binlog.inc @@ -1 +1,2 @@ +--source include/not_embedded.inc --source include/have_innodb.inc diff --git a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.test b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.test index 68d963da97545..f004b80c73d49 100644 --- a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.test +++ b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.test @@ -5,7 +5,6 @@ # binlogs (.ibb) into legacy-format binary log # ---source include/have_binlog_format_row.inc --source include/have_innodb_binlog.inc --let $datadir= `SELECT @@datadir` diff --git a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_max_size.result b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_max_size.result new file mode 100644 index 0000000000000..b69db14803a06 --- /dev/null +++ b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_max_size.result @@ -0,0 +1,15 @@ +include/reset_master.inc +CREATE TABLE t1 (a INT PRIMARY KEY, b LONGTEXT) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1, REPEAT('a', 48 * 1024)); +INSERT INTO t1 VALUES (2, REPEAT('b', 48 * 1024)); +INSERT INTO t1 VALUES (3, REPEAT('c', 48 * 1024)); +FLUSH BINARY LOGS; +*** Convert with a 32K maximum output binlog size +*** Rotated files form a replayable binlog sequence +DROP TABLE t1; +SELECT a, LENGTH(b) FROM t1 ORDER BY a; +a LENGTH(b) +1 49152 +2 49152 +3 49152 +DROP TABLE t1; diff --git a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_max_size.test b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_max_size.test new file mode 100644 index 0000000000000..841c72cdb108e --- /dev/null +++ b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_max_size.test @@ -0,0 +1,40 @@ +--source include/have_binlog_format_row.inc +--source include/have_innodb_binlog.inc + +--let $datadir= `SELECT @@datadir` + +--source include/reset_master.inc + + +CREATE TABLE t1 (a INT PRIMARY KEY, b LONGTEXT) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1, REPEAT('a', 48 * 1024)); +INSERT INTO t1 VALUES (2, REPEAT('b', 48 * 1024)); +INSERT INTO t1 VALUES (3, REPEAT('c', 48 * 1024)); + +# Complete binlog-000000.ibb and wait until it is available on disk. +FLUSH BINARY LOGS; +--let $binlog_name= binlog-000002.ibb +--let $binlog_size= 262144 +--source include/wait_for_engine_binlog.inc + +--echo *** Convert with a 32K maximum output binlog size +--exec $MYSQL_BINLOG --convert-engine-binlog --max-binlog-size=32768 --result-file=$MYSQL_TMP_DIR/max_size_conv $datadir/binlog-000000.ibb + +# Rotation is checked at transaction boundaries. The first large INSERT stays +# in the initial file; each following INSERT starts a new output file. +--file_exists $MYSQL_TMP_DIR/max_size_conv.000001 +--file_exists $MYSQL_TMP_DIR/max_size_conv.000002 +--file_exists $MYSQL_TMP_DIR/max_size_conv.000003 + +--echo *** Rotated files form a replayable binlog sequence +--exec $MYSQL_BINLOG --gtid-strict-mode=0 $MYSQL_TMP_DIR/max_size_conv.000001 $MYSQL_TMP_DIR/max_size_conv.000002 $MYSQL_TMP_DIR/max_size_conv.000003 > $MYSQLTEST_VARDIR/tmp/max_size_replay.sql +DROP TABLE t1; +--exec $MYSQL --abort-source-on-error -e "source $MYSQLTEST_VARDIR/tmp/max_size_replay.sql;" test +--remove_file $MYSQLTEST_VARDIR/tmp/max_size_replay.sql + +SELECT a, LENGTH(b) FROM t1 ORDER BY a; + +--remove_file $MYSQL_TMP_DIR/max_size_conv.000001 +--remove_file $MYSQL_TMP_DIR/max_size_conv.000002 +--remove_file $MYSQL_TMP_DIR/max_size_conv.000003 +DROP TABLE t1; From 027726363099f4f25d74192b94fab9da2bc51288 Mon Sep 17 00:00:00 2001 From: Tarun Wadhwa Date: Sun, 26 Jul 2026 16:17:04 +0000 Subject: [PATCH 05/14] Populate the GTID_LIST_EVENT from the FSP_BINLOG_TYPE_GTID_STATE stored in the 1st page of innodb based binlog --- client/mysqlbinlog-engine.cc | 104 ++++++++++++++++++ client/mysqlbinlog-engine.h | 3 +- client/mysqlbinlog.cc | 5 + ...nvert_engine_binlog_gtid_list_event.result | 9 +- ...convert_engine_binlog_gtid_list_event.test | 104 ++++++++++++++++-- 5 files changed, 214 insertions(+), 11 deletions(-) diff --git a/client/mysqlbinlog-engine.cc b/client/mysqlbinlog-engine.cc index 96421b35f2947..334bac6d8bb92 100644 --- a/client/mysqlbinlog-engine.cc +++ b/client/mysqlbinlog-engine.cc @@ -25,6 +25,8 @@ #include "mysqlbinlog-engine.h" #include "my_compr_int.h" #include "my_dir.h" +#include "rpl_gtid_base.h" + const char *INNODB_BINLOG_MAGIC= "\xfe\xfe\x0d\x01"; @@ -308,6 +310,7 @@ class binlog_reader_innodb : public handler_binlog_reader { uchar rd_buf[5*COMPR_INT_MAX64]; private: int read_data(uchar *buf, uint32_t len); + int read_gtid_state(rpl_binlog_state_base *state); public: binlog_reader_innodb(); @@ -322,6 +325,7 @@ class binlog_reader_innodb : public handler_binlog_reader { virtual void enable_single_file() override; bool is_valid() { return page_buf != nullptr; } bool init_from_fd_pos(File fd, ulonglong start_position); + bool get_initial_gtid_state(rpl_binlog_state_base *state); }; @@ -851,6 +855,100 @@ binlog_reader_innodb::init_from_fd_pos(File fd, ulonglong start_position) return false; } +bool binlog_reader_innodb::get_initial_gtid_state( + rpl_binlog_state_base *gtid_state) +{ + chunk_reader_mysqlbinlog::saved_position saved_pos; + + chunk_rd.save_pos(&saved_pos); + chunk_rd.seek(start_file_no, binlog_page_size); + + int res= read_gtid_state(gtid_state); + + chunk_rd.restore_pos(&saved_pos); + return res != 1; +} + +int binlog_reader_innodb::read_gtid_state(rpl_binlog_state_base *state) { + uchar buf[256]; + static_assert(sizeof(buf) >= 2*COMPR_INT_MAX64 + 6*COMPR_INT_MAX64, + "buf must hold at least 2 GTIDs"); + int res= chunk_rd.read_data(buf, sizeof(buf), true); + if (res < 0) + return -1; + if (res == 0 || chunk_rd.cur_type() != FSP_BINLOG_TYPE_GTID_STATE) + return 0; + + const uchar *p= buf; + const uchar *p_end= buf + res; + std::pair v_and_p= compr_int_read(p); + p= v_and_p.second; + if (p > p_end) + return -1; + uint64_t num_gtid= v_and_p.first; + + /* + The XA reference is part of the serialized GTID-state format and must be + consumed before decoding the GTID entries. mysqlbinlog does not use it, + since it is only needed by the server for XA recovery and binlog purging. + */ + v_and_p= compr_int_read(p); + p= v_and_p.second; + if (p > p_end) + return -1; + + /* Read each GTID one by one and add into the state. */ + for (uint64_t count= num_gtid; count > 0; --count) + { + ptrdiff_t remain= p_end - p; + /* Read more data as needed to ensure we have read a full GTID. */ + if (!chunk_rd.end_of_record() && + remain < 3*COMPR_INT_MAX64) + { + memmove(buf, p, remain); + res= chunk_rd.read_data(buf + remain, (int)(sizeof(buf) - remain), + true); + if (res < 0) + return -1; + p= buf; + p_end= p + remain + res; + remain+= res; + } + rpl_gtid gtid; + if (p >= p_end) + return -1; + v_and_p= compr_int_read(p); + if (v_and_p.first > UINT32_MAX) + return -1; + gtid.domain_id= (uint32_t)v_and_p.first; + p= v_and_p.second; + if (p >= p_end) + return -1; + v_and_p= compr_int_read(p); + if (v_and_p.first > UINT32_MAX) + return -1; + gtid.server_id= (uint32_t)v_and_p.first; + p= v_and_p.second; + if (p >= p_end) + return -1; + v_and_p= compr_int_read(p); + gtid.seq_no= v_and_p.first; + p= v_and_p.second; + if (p > p_end) + return -1; + if (state->update_nolock(>id)) + return -1; + } + + /* + For now, we expect no more data. + Later it could be extended, as we store (and read) the count of GTIDs. + */ + DBUG_ASSERT(p == p_end); + + return 1; +} + int binlog_reader_innodb::read_data(uchar *buf, uint32_t len) { @@ -1188,6 +1286,12 @@ open_engine_binlog(handler_binlog_reader *generic_reader, return reader->init_from_fd_pos(dup(opened_cache->file), start_position); } +bool read_initial_gtid_state(handler_binlog_reader *generic_reader, + rpl_binlog_state_base *gtid_state) +{ + binlog_reader_innodb *reader= (binlog_reader_innodb *) generic_reader; + return reader->get_initial_gtid_state(gtid_state); +} handler_binlog_reader * get_binlog_reader_innodb() diff --git a/client/mysqlbinlog-engine.h b/client/mysqlbinlog-engine.h index dbc4a85a5b1d9..af88b164c9e87 100644 --- a/client/mysqlbinlog-engine.h +++ b/client/mysqlbinlog-engine.h @@ -28,6 +28,7 @@ extern bool open_engine_binlog(handler_binlog_reader *reader, ulonglong start_position, const char *filename, IO_CACHE *opened_cache); - +extern bool read_initial_gtid_state(handler_binlog_reader *reader, + rpl_binlog_state_base *gtid_state); /* Shared functions defined in mysqlbinlog.cc */ extern void error(const char *format, ...) ATTRIBUTE_FORMAT(printf, 1, 2); diff --git a/client/mysqlbinlog.cc b/client/mysqlbinlog.cc index c5f13c2cf94fb..1c8088fa1828a 100644 --- a/client/mysqlbinlog.cc +++ b/client/mysqlbinlog.cc @@ -3946,6 +3946,11 @@ static Exit_status dump_local_log_entries(PRINT_EVENT_INFO *print_event_info, if (opt_convert_engine_binlog && !gtid_state) { gtid_state= new rpl_binlog_state_base(); gtid_state->init(); + /* initialize this gtid state from the header of the innodb binlog file */ + if (read_initial_gtid_state(engine_binlog_reader, gtid_state)) { + error("Failed to read initial GTID state from the header of the innodb binlog file"); + goto err; + } } } for (;;) diff --git a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_gtid_list_event.result b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_gtid_list_event.result index f31206ace645f..718fed551fe55 100644 --- a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_gtid_list_event.result +++ b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_gtid_list_event.result @@ -40,9 +40,16 @@ FOUND 1 /Rotate to gtid_conv.000004/ in gtid_conv_listing.txt FOUND 1 /Gtid list \[0-1-2,\n# 1-2-1,\n# 1-1-2,\n# 2-3-1,\n# 3-4-1\]/ in gtid_conv_listing.txt *** Convert the generated InnoDB binlog(binlog-000002.ibb) to legacy format in random order *** Verify synthesized Gtid_list events in converted legacy binlogs -NOT FOUND /Gtid list \[0-1-2,\n# 1-2-1,\n# 1-1-2,\n# 2-3-1,\n# 3-4-1\]/ in gtid_conv_listing.txt +FOUND 1 /Gtid list \[0-1-2,\n# 1-2-1,\n# 2-3-1\]/ in gtid_conv_listing.txt FOUND 1 /Rotate to gtid_conv.000002/ in gtid_conv_listing.txt +FOUND 1 /Gtid list \[0-1-2,\n# 1-2-1,\n# 1-1-2,\n# 2-3-1,\n# 3-4-1\]/ in gtid_conv_listing.txt DROP TABLE t1; DROP TABLE t2; DROP TABLE t3; DROP TABLE t4; +*** Verify a large initial GTID state +include/reset_master.inc +FOUND 1 /Gtid list \[10-1-1,/ in large_gtid_conv_listing.txt +FOUND 1 /70-1-1,/ in large_gtid_conv_listing.txt +FOUND 1 /80-1-1,/ in large_gtid_conv_listing.txt +FOUND 1 /109-1-1\]/ in large_gtid_conv_listing.txt diff --git a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_gtid_list_event.test b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_gtid_list_event.test index 721a37318e0e2..fa908dde97d69 100644 --- a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_gtid_list_event.test +++ b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_gtid_list_event.test @@ -5,14 +5,42 @@ # Also verify that the generated legacy binlogs contain the expected ROTATE_EVENT. # # Methodology: -# Generate several .ibb files with GTIDs across multiple domains and server -# ids, including GTIDs that are not ordered by server id. Restart the server -# in the middle so the converter sees another FORMAT_DESCRIPTION_EVENT and -# rotates the generated legacy output. Convert all completed .ibb files and -# verify each generated legacy binlog contains the expected Gtid_list state. -# Convert the .ibb files again in random order and verify the generated -# legacy binlogs contain the expected Gtid_list state. +# This test contains two scenarios: # +# 1. Multiple binlog files, domains, server IDs, and server restart: +# Generate GTIDs across several InnoDB-format binlog files using multiple +# domain IDs and server IDs, including GTIDs not ordered by server ID. +# Restart the server so the converter encounters another +# FORMAT_DESCRIPTION_EVENT and rotates the generated legacy output. +# Convert the completed .ibb files and verify that every generated legacy +# binlog starts with the expected synthesized GTID_LIST_EVENT. Also convert +# a later .ibb file independently and verify that its initial GTID state is +# recovered from the InnoDB binlog header and preserved across the output +# rotation. +# +# 2. Large initial GTID state: +# Generate a large list of GTIDs in distinct domains, rotate the InnoDB +# binlog, and convert the later .ibb file independently. Verify entries at +# the beginning, middle, and end of the synthesized GTID_LIST_EVENT to +# confirm that the complete initial GTID state is decoded correctly. +# + +#### Test case 1 #### +# +# Summary of GTIDs generated by case 1: +# +# Input InnoDB binlog Transaction Generated GTID +# -------------------- -------------------------- -------------- +# binlog-000000.ibb INSERT INTO t1 VALUES (2) 0-1-1 +# binlog-000000.ibb INSERT INTO t2 VALUES (1) 1-2-1 +# binlog-000001.ibb INSERT INTO t1 VALUES (1) 0-1-2 +# binlog-000001.ibb INSERT INTO t3 VALUES (1) 2-3-1 +# binlog-000002.ibb INSERT INTO t2 VALUES (2) 1-1-2 +# binlog-000002.ibb INSERT INTO t4 VALUES (1) 3-4-1 +# server restart +# binlog-000002.ibb INSERT INTO t1 VALUES (3) 0-1-3 +# + --source include/have_binlog_format_row.inc --source include/have_innodb_binlog.inc @@ -122,18 +150,76 @@ FLUSH BINARY LOGS; --echo *** Verify synthesized Gtid_list events in converted legacy binlogs --exec $MYSQL_BINLOG --verbose $MYSQL_TMP_DIR/gtid_conv.000001 > $MYSQL_TMP_DIR/gtid_conv_listing.txt --let SEARCH_FILE= $MYSQL_TMP_DIR/gtid_conv_listing.txt ---let SEARCH_PATTERN= Gtid list \[0-1-2,\n# 1-2-1,\n# 1-1-2,\n# 2-3-1,\n# 3-4-1\] -# TODO: Tarun this search will fail now as this functionality is not yet implemented +--let SEARCH_PATTERN= Gtid list \[0-1-2,\n# 1-2-1,\n# 2-3-1\] --source include/search_pattern_in_file.inc --let SEARCH_PATTERN= Rotate to gtid_conv.000002 --source include/search_pattern_in_file.inc +--exec $MYSQL_BINLOG --verbose $MYSQL_TMP_DIR/gtid_conv.000002 > $MYSQL_TMP_DIR/gtid_conv_listing.txt +--let SEARCH_FILE= $MYSQL_TMP_DIR/gtid_conv_listing.txt +--let SEARCH_PATTERN= Gtid list \[0-1-2,\n# 1-2-1,\n# 1-1-2,\n# 2-3-1,\n# 3-4-1\] +--source include/search_pattern_in_file.inc + --remove_file $MYSQL_TMP_DIR/gtid_conv_listing.txt --remove_file $MYSQL_TMP_DIR/gtid_conv.000001 +--remove_file $MYSQL_TMP_DIR/gtid_conv.000002 DROP TABLE t1; DROP TABLE t2; DROP TABLE t3; DROP TABLE t4; + +#### Test case 2 #### + +--echo *** Verify a large initial GTID state +--disable_query_log +--source include/reset_master.inc + +SET @@session.sql_log_bin= 0; +CREATE TABLE t_large_gtid_state (a INT PRIMARY KEY) ENGINE=InnoDB; +SET @@session.sql_log_bin= 1; +SET @@session.server_id= 1; + +# Generate 100 distinct GTID-state entries. +--let $domain_id= 10 +while ($domain_id < 110) { + --eval SET @@session.gtid_domain_id= $domain_id + --eval INSERT INTO t_large_gtid_state VALUES ($domain_id) + --inc $domain_id +} + +# The initial GTID state of binlog-000001.ibb contains the 100 entries above. +# Add an event so converting that file creates a legacy output file, then +# rotate once more to ensure the input file is complete. +FLUSH BINARY LOGS; +SET @@session.gtid_domain_id= 200; +INSERT INTO t_large_gtid_state VALUES (200); +FLUSH BINARY LOGS; + +--let $binlog_name= binlog-000002.ibb +--let $binlog_size= 262144 +--source include/wait_for_engine_binlog.inc +--enable_query_log + +--exec $MYSQL_BINLOG --convert-engine-binlog --result-file=$MYSQL_TMP_DIR/large_gtid_conv $datadir/binlog-000001.ibb +--exec $MYSQL_BINLOG --verbose $MYSQL_TMP_DIR/large_gtid_conv.000001 > $MYSQL_TMP_DIR/large_gtid_conv_listing.txt +--let SEARCH_FILE= $MYSQL_TMP_DIR/large_gtid_conv_listing.txt + + +--let SEARCH_PATTERN= Gtid list \[10-1-1, +--source include/search_pattern_in_file.inc +--let SEARCH_PATTERN= 70-1-1, +--source include/search_pattern_in_file.inc +--let SEARCH_PATTERN= 80-1-1, +--source include/search_pattern_in_file.inc +--let SEARCH_PATTERN= 109-1-1\] +--source include/search_pattern_in_file.inc + +--remove_file $MYSQL_TMP_DIR/large_gtid_conv_listing.txt +--remove_file $MYSQL_TMP_DIR/large_gtid_conv.000001 + +--disable_query_log +DROP TABLE t_large_gtid_state; +--enable_query_log From a1d0af16665a3b90fec27475b1888bdd0b4743f5 Mon Sep 17 00:00:00 2001 From: Tarun Wadhwa Date: Sun, 26 Jul 2026 17:20:47 +0000 Subject: [PATCH 06/14] Updated the max value for max-binlog-size flag to be 4GB. Also replaced the errors with warnings if end_log_pos exceeds the 4GB mark --- client/mysqlbinlog.cc | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/client/mysqlbinlog.cc b/client/mysqlbinlog.cc index 1c8088fa1828a..da4760b1f5d96 100644 --- a/client/mysqlbinlog.cc +++ b/client/mysqlbinlog.cc @@ -129,7 +129,7 @@ static my_bool debug_info_flag, debug_check_flag; static my_bool force_if_open_opt= 1; static my_bool opt_raw_mode= 0, opt_stop_never= 0; static my_bool opt_convert_engine_binlog= 0; -static ulong opt_max_binlog_size= 1024L * 1024L * 1024L; +static ulonglong opt_max_binlog_size= 1024ULL * 1024ULL * 1024ULL; my_bool opt_gtid_strict_mode= true; static ulong opt_stop_never_slave_server_id= 0; static my_bool opt_verify_binlog_checksum= 1; @@ -221,6 +221,7 @@ static char out_file_name[FN_REFLEN + 1]= {0}; /* Used to track the position of the log file for computing legacy end_log_pos for converted legacy binlog file */ static ulonglong log_file_pos= 0; +static bool log_file_pos_overflow_warning_printed= false; static rpl_binlog_state_base *gtid_state= NULL; @@ -1575,16 +1576,16 @@ Exit_status process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev, ############################################################################## */ -static bool store_log_file_pos(uchar *pos) +static void store_log_file_pos(uchar *pos) { - if (log_file_pos > UINT_MAX32) + if (log_file_pos > UINT_MAX32 && !log_file_pos_overflow_warning_printed) { - error("Converted binlog output file exceeds the maximum supported size of " - "4GB"); - return true; + warning("Converted binlog output file '%s' exceeds 4GB; event end_log_pos " + "values might be corrupted", + out_file_name); + log_file_pos_overflow_warning_printed= true; } int4store(pos, log_file_pos); - return false; } static bool write_event_header(FILE *outfile, Log_event_type event_type, @@ -1617,8 +1618,7 @@ static bool write_event_header(FILE *outfile, Log_event_type event_type, int2store(header + FLAGS_OFFSET, 0); /* Update the log_file_pos */ log_file_pos+= event_len; - if (store_log_file_pos(header + LOG_POS_OFFSET)) - return true; + store_log_file_pos(header + LOG_POS_OFFSET); /* Write this header to outfile */ if (my_fwrite(outfile, (const uchar *)header, LOG_EVENT_HEADER_LEN, MYF(MY_NABP))) { @@ -1661,8 +1661,7 @@ static bool update_event_end_log_pos(Log_event *ev) DBUG_ASSERT(ev->temp_buf != NULL && ev->data_written >= LOG_EVENT_HEADER_LEN); log_file_pos+= ev->data_written; - if (store_log_file_pos(ev->temp_buf + LOG_POS_OFFSET)) - return true; + store_log_file_pos(ev->temp_buf + LOG_POS_OFFSET); return false; } @@ -1946,6 +1945,7 @@ static bool init_output_legacy_binlog(FILE **out_file, char *out_name, { /* Reset the log_file_pos to 0 for the new output legacy binlog file */ log_file_pos= 0; + log_file_pos_overflow_warning_printed= false; generate_output_legacy_binlog_name(out_name, out_name_len, ++convert_engine_output_index); @@ -2169,11 +2169,10 @@ static struct my_option my_options[] = "Convert InnoDB based engine binlog files to legacy binlog files.", &opt_convert_engine_binlog, &opt_convert_engine_binlog, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - /* TODO: Tarun what should be the max value? */ {"max-binlog-size", 0, "Maximum size of converted legacy binlog files.", - &opt_max_binlog_size, &opt_max_binlog_size, 0, GET_ULONG, REQUIRED_ARG, - 1024L * 1024L * 1024L, IO_SIZE, 1024L * 1024L * 1024L, 0, IO_SIZE, 0}, + &opt_max_binlog_size, &opt_max_binlog_size, 0, GET_ULL, REQUIRED_ARG, + 1024ULL * 1024ULL * 1024ULL, IO_SIZE, 4ULL * 1024ULL * 1024ULL * 1024ULL, 0, IO_SIZE, 0}, {"flashback", 'B', "Flashback feature can rollback you committed data to a special time point.", #ifdef WHEN_FLASHBACK_REVIEW_READY "before Flashback feature writing a row, original row can insert to review-dbname.review-tablename," From bff955c5e5551047c918f23b3539e8064debcb1e Mon Sep 17 00:00:00 2001 From: Tarun Wadhwa Date: Mon, 27 Jul 2026 08:40:33 +0000 Subject: [PATCH 07/14] Reverted some changes required for creation of Binlog_checkpoint_log_event object. And instead directly passed the filename to generate its serialized version --- client/mysqlbinlog.cc | 29 ++++++++----------- ...lbinlog_convert_engine_binlog_basic.result | 2 +- ...sqlbinlog_convert_engine_binlog_basic.test | 2 +- sql/log_event.cc | 10 ------- sql/log_event.h | 5 ++-- sql/log_event_server.cc | 11 +++++++ 6 files changed, 27 insertions(+), 32 deletions(-) diff --git a/client/mysqlbinlog.cc b/client/mysqlbinlog.cc index da4760b1f5d96..b1b04ce6057fe 100644 --- a/client/mysqlbinlog.cc +++ b/client/mysqlbinlog.cc @@ -1812,14 +1812,15 @@ static bool write_gtid_list_event_to_legacy_binlog(FILE *outfile, return false; } -/* - TODO: Tarun get this reviewed. should i pass the event to this function - or alternatively i can just pass the required args - (binlog_file_name, binlog_file_len) to this function - and we don't have to extract the constructor for BINLOG_CHECKPOINT_EVENT +/* + Writes the BINLOG_CHECKPOINT_EVENT to the output legacy binlog file. + @param outfile: The output legacy binlog file + @param filename: The basename of the binlog file + @param length: The length of the basename + @return: true if failed, false if successful */ static bool write_binlog_checkpoint_event_to_legacy_binlog( - FILE *outfile, Binlog_checkpoint_log_event *bcle) + FILE *outfile, const char *filename, uint32 length) { my_bool do_checksum; ha_checksum crc= 0; @@ -1831,14 +1832,14 @@ static bool write_binlog_checkpoint_event_to_legacy_binlog( to_packet() method used in case of FORMAT_DESCRIPTION_EVENT, GTID_LIST_EVENT, etc.) */ uchar header_buf[BINLOG_CHECKPOINT_HEADER_LEN]; - int4store(header_buf, bcle->binlog_file_len); + int4store(header_buf, length); if (str.append((char *) header_buf, BINLOG_CHECKPOINT_HEADER_LEN)) { error("Failed due to out-of-memory writing BINLOG_CHECKPOINT_EVENT body"); return true; } - if (str.append((char *) bcle->binlog_file_name, bcle->binlog_file_len)) + if (str.append((char *) filename, length)) { error("Failed due to out-of-memory writing BINLOG_CHECKPOINT_EVENT body"); return true; @@ -1988,15 +1989,9 @@ static bool init_output_legacy_binlog(FILE **out_file, char *out_name, size_t off= dirname_length(out_name); uint32 length= (uint32) (strlen(out_name) - off); - // Write the BINLOG_CHECKPOINT_EVENT to the output legacy binlog file - Binlog_checkpoint_log_event bcle= - Binlog_checkpoint_log_event(out_name + off, length); - if (!bcle.is_valid()) - { - error("Failed to create BINLOG_CHECKPOINT_EVENT"); - return true; - } - if (write_binlog_checkpoint_event_to_legacy_binlog(*out_file, &bcle)) + /* Write the BINLOG_CHECKPOINT_EVENT to the output legacy binlog file */ + if (write_binlog_checkpoint_event_to_legacy_binlog(*out_file, out_name + off, + length)) { error("Could not write BINLOG_CHECKPOINT_EVENT to output legacy binlog " "file"); diff --git a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.result b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.result index 74fa34cc64c52..cc82bc2a69174 100644 --- a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.result +++ b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.result @@ -14,7 +14,7 @@ FLUSH BINARY LOGS; *** Convert binlog-000000.ibb to legacy format *** Converted file contains the synthesized header events FOUND 1 /Gtid list/ in conv_listing.txt -FOUND 1 /Binlog checkpoint/ in conv_listing.txt +FOUND 1 /Binlog checkpoint conv.000001/ in conv_listing.txt FOUND 1 /Start: binlog v 4/ in conv_listing.txt NOT FOUND /Rotate to/ in conv_listing.txt *** Round-trip: replay the converted file and compare data diff --git a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.test b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.test index f004b80c73d49..4da28010f3e04 100644 --- a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.test +++ b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.test @@ -40,7 +40,7 @@ FLUSH BINARY LOGS; --let SEARCH_FILE= $MYSQL_TMP_DIR/conv_listing.txt --let SEARCH_PATTERN= Gtid list --source include/search_pattern_in_file.inc ---let SEARCH_PATTERN= Binlog checkpoint +--let SEARCH_PATTERN= Binlog checkpoint conv.000001 --source include/search_pattern_in_file.inc --let SEARCH_PATTERN= Start: binlog v 4 --source include/search_pattern_in_file.inc diff --git a/sql/log_event.cc b/sql/log_event.cc index 3e29d057a107d..90a4ebe1d4ef5 100644 --- a/sql/log_event.cc +++ b/sql/log_event.cc @@ -2614,16 +2614,6 @@ Rotate_log_event::Rotate_log_event(const uchar *buf, uint event_len, Binlog_checkpoint_log_event methods **************************************************************************/ -Binlog_checkpoint_log_event::Binlog_checkpoint_log_event( - const char *binlog_file_name_arg, uint binlog_file_len_arg) - : Log_event(), - binlog_file_name(my_strndup(PSI_INSTRUMENT_ME, binlog_file_name_arg, - binlog_file_len_arg, MYF(MY_WME))), - binlog_file_len(binlog_file_len_arg) -{ - cache_type= EVENT_NO_CACHE; -} - Binlog_checkpoint_log_event::Binlog_checkpoint_log_event( const uchar *buf, uint event_len, const Format_description_log_event *description_event) diff --git a/sql/log_event.h b/sql/log_event.h index f2edc379334f1..b7e365fea1edd 100644 --- a/sql/log_event.h +++ b/sql/log_event.h @@ -3330,15 +3330,14 @@ class Binlog_checkpoint_log_event: public Log_event uint binlog_file_len; #ifdef MYSQL_SERVER - + Binlog_checkpoint_log_event(const char *binlog_file_name_arg, + uint binlog_file_len_arg); #ifdef HAVE_REPLICATION void pack_info(Protocol *protocol) override; #endif #else bool print(FILE *file, PRINT_EVENT_INFO *print_event_info) override; #endif - Binlog_checkpoint_log_event(const char *binlog_file_name_arg, - uint binlog_file_len_arg); Binlog_checkpoint_log_event(const uchar *buf, uint event_len, const Format_description_log_event *description_event); diff --git a/sql/log_event_server.cc b/sql/log_event_server.cc index 7424e4794d17c..c1fd9d355c766 100644 --- a/sql/log_event_server.cc +++ b/sql/log_event_server.cc @@ -2830,6 +2830,17 @@ Binlog_checkpoint_log_event::do_shall_skip(rpl_group_info *rgi) #endif +Binlog_checkpoint_log_event::Binlog_checkpoint_log_event( + const char *binlog_file_name_arg, + uint binlog_file_len_arg) + :Log_event(), + binlog_file_name(my_strndup(PSI_INSTRUMENT_ME, binlog_file_name_arg, binlog_file_len_arg, + MYF(MY_WME))), + binlog_file_len(binlog_file_len_arg) +{ + cache_type= EVENT_NO_CACHE; +} + bool Binlog_checkpoint_log_event::write(Log_event_writer *writer) { From 2601af0577f2f447bd8998068406c4ffa8623ec8 Mon Sep 17 00:00:00 2001 From: Tarun Wadhwa Date: Thu, 30 Jul 2026 19:59:12 +0000 Subject: [PATCH 08/14] Minor refactoring related to opt_convert_engine_binlog --- client/mysqlbinlog.cc | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/client/mysqlbinlog.cc b/client/mysqlbinlog.cc index b1b04ce6057fe..11605d925c71f 100644 --- a/client/mysqlbinlog.cc +++ b/client/mysqlbinlog.cc @@ -4135,7 +4135,24 @@ int main(int argc, char** argv) "with --flashback"); die(1); } - + if (stop_datetime_given) + { + error("The --convert-engine-binlog option cannot be combined " + "with --stop-datetime"); + die(1); + } + if (static_cast(stop_position) != stop_position_default) + { + error("The --convert-engine-binlog option cannot be combined " + "with --stop-position"); + die(1); + } + if (position_gtid_filter) + { + error("The --convert-engine-binlog option does not support GTID values " + "for --start-position or --stop-position"); + die(1); + } } else if (opt_raw_mode) { @@ -4237,16 +4254,14 @@ int main(int argc, char** argv) if we finished processing input before reaching the stop boundaries indicated by --stop-datetime or --stop-position. */ - if (!opt_convert_engine_binlog && - stop_datetime_given && stop_datetime > last_processed_ev.datetime) + if (stop_datetime_given && stop_datetime > last_processed_ev.datetime) warning("Did not reach stop datetime '%s' before end of input", stop_datetime_str); - if (!opt_convert_engine_binlog && - (static_cast(stop_position) != stop_position_default) && + if ((static_cast(stop_position) != stop_position_default) && stop_position > last_processed_ev.position) warning("Did not reach stop position %llu before end of input", stop_position); - if (!opt_convert_engine_binlog && position_gtid_filter) + if (position_gtid_filter) position_gtid_filter->verify_final_state(); /* From 06526f673f7f2ca94f4dd071bd4c049b25306095 Mon Sep 17 00:00:00 2001 From: Tarun Wadhwa Date: Tue, 4 Aug 2026 12:25:35 +0000 Subject: [PATCH 09/14] Added changes to populate server_id and timestamp for synthetically generated events from the first event --- client/mysqlbinlog.cc | 65 ++++++++++--------- ...lbinlog_convert_engine_binlog_basic.result | 4 ++ ...sqlbinlog_convert_engine_binlog_basic.test | 10 +++ 3 files changed, 50 insertions(+), 29 deletions(-) diff --git a/client/mysqlbinlog.cc b/client/mysqlbinlog.cc index 11605d925c71f..df909ec8be2f4 100644 --- a/client/mysqlbinlog.cc +++ b/client/mysqlbinlog.cc @@ -219,12 +219,17 @@ static char default_output_legacy_binlog_prefix[]= "legacy_log"; static char out_file_name[FN_REFLEN + 1]= {0}; -/* Used to track the position of the log file for computing legacy end_log_pos for converted legacy binlog file */ +/* Used to track the position of the log file for computing legacy end_log_pos + * for converted legacy binlog file */ static ulonglong log_file_pos= 0; static bool log_file_pos_overflow_warning_printed= false; static rpl_binlog_state_base *gtid_state= NULL; +/* Used for synthetically generated event headers while converting innodb + * binlog to legacy */ +static uint32 generated_event_timestamp; +static uint32 generated_event_server_id; /** Pointer to the last read Annotate_rows_log_event. Having read an @@ -1590,7 +1595,8 @@ static void store_log_file_pos(uchar *pos) static bool write_event_header(FILE *outfile, Log_event_type event_type, ulong extra_len, time_t timestamp, - my_bool *do_checksum, ha_checksum *crc, + uint32 server_id, my_bool *do_checksum, + ha_checksum *crc, enum_binlog_checksum_alg checksum_alg) { uchar header[LOG_EVENT_HEADER_LEN]; @@ -1600,20 +1606,18 @@ static bool write_event_header(FILE *outfile, Log_event_type event_type, checksum_alg != BINLOG_CHECKSUM_ALG_UNDEF; int4store(header, timestamp); - header[EVENT_TYPE_OFFSET] = (uchar)event_type; - event_len= LOG_EVENT_HEADER_LEN + extra_len + - (*do_checksum ? BINLOG_CHECKSUM_LEN : 0); + header[EVENT_TYPE_OFFSET]= (uchar) event_type; + event_len= LOG_EVENT_HEADER_LEN + extra_len + + (*do_checksum ? BINLOG_CHECKSUM_LEN : 0); - // TODO: Tarun get this reviewed. what should the server id be? should i get from - // innodb binlog header - int4store(header + SERVER_ID_OFFSET, 0); + int4store(header + SERVER_ID_OFFSET, server_id); int4store(header + EVENT_LEN_OFFSET, event_len); /* - Notes: For a normal open/current binlog file, the format-description - header flags are typically 0x0001. After clean close, they become - 0x0000. - For GTID_LIST_EVENT and BINLOG_CHECKPOINT_EVENT, the flags are 0x0000 typically. - TODO: Tarun get this reviewed. + Notes: For a normal open/current binlog file, the format-description + header flags are typically 0x0001. After clean close, they become + 0x0000. + For GTID_LIST_EVENT and BINLOG_CHECKPOINT_EVENT, the flags are 0x0000 + typically. */ int2store(header + FLAGS_OFFSET, 0); /* Update the log_file_pos */ @@ -1621,19 +1625,20 @@ static bool write_event_header(FILE *outfile, Log_event_type event_type, store_log_file_pos(header + LOG_POS_OFFSET); /* Write this header to outfile */ - if (my_fwrite(outfile, (const uchar *)header, LOG_EVENT_HEADER_LEN, MYF(MY_NABP))) { + if (my_fwrite(outfile, (const uchar *) header, LOG_EVENT_HEADER_LEN, + MYF(MY_NABP))) + { error("Could not write header into converted binlog file '%s'", out_file_name); return true; } if (*do_checksum) { - *crc= my_checksum(0, (uchar*)header, sizeof(header)); + *crc= my_checksum(0, (uchar *) header, sizeof(header)); } return false; } - static bool write_event_footer(FILE *outfile, my_bool do_checksum, ha_checksum crc) { @@ -1723,10 +1728,11 @@ static bool write_format_description_event_to_legacy_binlog( error("Failed due to out-of-memory writing Format_description event"); return true; } - /* Write header of FORMAT_DESCRIPTION_EVENT to output legacy binlog file first */ + /* Write header of FORMAT_DESCRIPTION_EVENT to output legacy binlog file + * first */ if (write_event_header(outfile, FORMAT_DESCRIPTION_EVENT, str.length(), - fdev->created, &do_checksum, &crc, - BINLOG_CHECKSUM_ALG_CRC32)) + generated_event_timestamp, generated_event_server_id, + &do_checksum, &crc, BINLOG_CHECKSUM_ALG_CRC32)) { error("Could not write FORMAT_DESCRIPTION_EVENT header to output legacy " "binlog file"); @@ -1775,9 +1781,8 @@ static bool write_gtid_list_event_to_legacy_binlog(FILE *outfile, } /* Write header of GTID_LIST_EVENT to output legacy binlog file first */ - /* TODO: Tarun verify and fix behaviour of timestamp (ts) */ - time_t ts= 0; - if (write_event_header(outfile, GTID_LIST_EVENT, str.length(), ts, + if (write_event_header(outfile, GTID_LIST_EVENT, str.length(), + generated_event_timestamp, generated_event_server_id, &do_checksum, &crc, BINLOG_CHECKSUM_ALG_OFF)) { error( @@ -1847,9 +1852,8 @@ static bool write_binlog_checkpoint_event_to_legacy_binlog( /* Write header of BINLOG_CHECKPOINT_EVENT to output legacy binlog file first */ - /* TODO: Tarun verify and fix behaviour of timestamp (ts) */ - time_t ts= 0; - if (write_event_header(outfile, BINLOG_CHECKPOINT_EVENT, str.length(), ts, + if (write_event_header(outfile, BINLOG_CHECKPOINT_EVENT, str.length(), + generated_event_timestamp, generated_event_server_id, &do_checksum, &crc, BINLOG_CHECKSUM_ALG_OFF)) { error("Could not write BINLOG_CHECKPOINT_EVENT header to output legacy " @@ -1895,10 +1899,9 @@ write_rotate_log_event_to_legacy_binlog(FILE *outfile, uint ident_len= (uint) strlen(p); /* Write header of ROTATE_EVENT */ - /* TODO: Tarun handle the timestamp */ - time_t ts= 0; if (write_event_header(outfile, ROTATE_EVENT, ident_len + ROTATE_HEADER_LEN, - ts, &do_checksum, &crc, BINLOG_CHECKSUM_ALG_OFF)) + generated_event_timestamp, generated_event_server_id, + &do_checksum, &crc, BINLOG_CHECKSUM_ALG_OFF)) { error("Could not write ROTATE_EVENT header to output legacy binlog file"); return true; @@ -2024,6 +2027,11 @@ static bool rotate_output_legacy_binlog(FILE **out_file, char *out_name, */ static Exit_status write_event_to_legacy_binlog(Log_event *ev) { + /* + Update the global server_id and timestamp variables + */ + generated_event_server_id= ev->server_id; + generated_event_timestamp= (uint32) ev->when; // if event type is FORMAT_DESCRIPTION_EVENT, store the event in global // variable glob_description_event @@ -2051,7 +2059,6 @@ static Exit_status write_event_to_legacy_binlog(Log_event *ev) */ if (!output_legacy_binlog_file) { - if (init_output_legacy_binlog(&output_legacy_binlog_file, out_file_name, sizeof(out_file_name))) goto err; diff --git a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.result b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.result index cc82bc2a69174..64207c59f6cc5 100644 --- a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.result +++ b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.result @@ -1,5 +1,6 @@ include/reset_master.inc set TIMESTAMP= UNIX_TIMESTAMP("1970-01-21 15:32:22"); +SET SESSION server_id= 12345; *** Generate a small workload into binlog-000000.ibb CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE=InnoDB; INSERT INTO t1 VALUES (1, 0), (2, 0), (3, 0); @@ -16,6 +17,9 @@ FLUSH BINARY LOGS; FOUND 1 /Gtid list/ in conv_listing.txt FOUND 1 /Binlog checkpoint conv.000001/ in conv_listing.txt FOUND 1 /Start: binlog v 4/ in conv_listing.txt +FOUND 1 /#700121 15:32:22 server id 12345.*Start: binlog v 4/ in conv_listing.txt +FOUND 1 /#700121 15:32:22 server id 12345.*Gtid list/ in conv_listing.txt +FOUND 1 /#700121 15:32:22 server id 12345.*Binlog checkpoint conv.000001/ in conv_listing.txt NOT FOUND /Rotate to/ in conv_listing.txt *** Round-trip: replay the converted file and compare data DROP TABLE t1; diff --git a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.test b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.test index 4da28010f3e04..0e2c297f6dcbd 100644 --- a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.test +++ b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.test @@ -14,6 +14,7 @@ # Fixed timestamp for deterministic event listings (with --timezone=GMT-3 # from the -master.opt file, matching binlog_in_engine.mysqlbinlog). set TIMESTAMP= UNIX_TIMESTAMP("1970-01-21 15:32:22"); +SET SESSION server_id= 12345; --echo *** Generate a small workload into binlog-000000.ibb CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE=InnoDB; @@ -44,6 +45,15 @@ FLUSH BINARY LOGS; --source include/search_pattern_in_file.inc --let SEARCH_PATTERN= Start: binlog v 4 --source include/search_pattern_in_file.inc +# The three synthesized events must inherit the timestamp and server ID of the +# first input event instead of receiving zero-valued headers. +--let SEARCH_PATTERN= #700121 15:32:22 server id 12345.*Start: binlog v 4 +--source include/search_pattern_in_file.inc +--let SEARCH_PATTERN= #700121 15:32:22 server id 12345.*Gtid list +--source include/search_pattern_in_file.inc +--let SEARCH_PATTERN= #700121 15:32:22 server id 12345.*Binlog checkpoint conv.000001 +--source include/search_pattern_in_file.inc + # it should not contain the Rotate event in the converted file --let SEARCH_PATTERN= Rotate to --source include/search_pattern_in_file.inc From cf323624c10d4ebecd2b68fa4ed72b9c041b7dfd Mon Sep 17 00:00:00 2001 From: Tarun Wadhwa Date: Thu, 6 Aug 2026 20:25:03 +0000 Subject: [PATCH 10/14] Use the current build's server version for synthesized format description event --- client/mysqlbinlog.cc | 13 +++++++------ .../mysqlbinlog_convert_engine_binlog_basic.result | 1 + .../mysqlbinlog_convert_engine_binlog_basic.test | 2 ++ sql/log_event.cc | 1 - sql/rpl_gtid.cc | 2 -- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/client/mysqlbinlog.cc b/client/mysqlbinlog.cc index df909ec8be2f4..bb57d3cfbe0b1 100644 --- a/client/mysqlbinlog.cc +++ b/client/mysqlbinlog.cc @@ -1797,11 +1797,6 @@ static bool write_gtid_list_event_to_legacy_binlog(FILE *outfile, return true; } - /* - TODO: Tarun get this reviewed. We are disabling/not supporting the checksum in every event. - Should I keep the below code for extensibility? - (do_checksum will always be false here) - */ if (do_checksum) { crc= my_checksum(crc, (uchar*)str.ptr(), str.length()); } @@ -3661,6 +3656,13 @@ static Exit_status check_header(IO_CACHE* file, int read_error; delete glob_description_event; + /* + Use the current build's server version in the synthesized + FORMAT_DESCRIPTION_EVENT written during engine-binlog conversion. + */ + if (opt_convert_engine_binlog) + strmake(server_version, MYSQL_SERVER_VERSION, sizeof(server_version) - 1); + if (!(glob_description_event= new Format_description_log_event(4))) { error("Failed creating Format_description_log_event; out of memory?"); @@ -4048,7 +4050,6 @@ static Exit_status dump_local_log_entries(PRINT_EVENT_INFO *print_event_info, end: if (output_legacy_binlog_file) { - /* TODO: Tarun write the STOP_EVENT maybe? */ my_fclose(output_legacy_binlog_file, MYF(0)); output_legacy_binlog_file= NULL; } diff --git a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.result b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.result index 64207c59f6cc5..d2c73dc5adcb8 100644 --- a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.result +++ b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.result @@ -17,6 +17,7 @@ FLUSH BINARY LOGS; FOUND 1 /Gtid list/ in conv_listing.txt FOUND 1 /Binlog checkpoint conv.000001/ in conv_listing.txt FOUND 1 /Start: binlog v 4/ in conv_listing.txt +FOUND 1 /server v 13.1.0-MariaDB/ in conv_listing.txt FOUND 1 /#700121 15:32:22 server id 12345.*Start: binlog v 4/ in conv_listing.txt FOUND 1 /#700121 15:32:22 server id 12345.*Gtid list/ in conv_listing.txt FOUND 1 /#700121 15:32:22 server id 12345.*Binlog checkpoint conv.000001/ in conv_listing.txt diff --git a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.test b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.test index 0e2c297f6dcbd..815953941f8b7 100644 --- a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.test +++ b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.test @@ -45,6 +45,8 @@ FLUSH BINARY LOGS; --source include/search_pattern_in_file.inc --let SEARCH_PATTERN= Start: binlog v 4 --source include/search_pattern_in_file.inc +--let SEARCH_PATTERN= server v $MYSQL_SERVER_VERSION +--source include/search_pattern_in_file.inc # The three synthesized events must inherit the timestamp and server ID of the # first input event instead of receiving zero-valued headers. --let SEARCH_PATTERN= #700121 15:32:22 server id 12345.*Start: binlog v 4 diff --git a/sql/log_event.cc b/sql/log_event.cc index 90a4ebe1d4ef5..17d5741ed8236 100644 --- a/sql/log_event.cc +++ b/sql/log_event.cc @@ -2384,7 +2384,6 @@ Format_description_log_event::to_packet(String *packet) p += 2; memcpy(p, server_version, ST_SERVER_VER_LEN); p+= ST_SERVER_VER_LEN; -/* TODO: Tarun get this reviewed */ #ifdef MYSQL_SERVER if (!dont_set_created) created= get_time(); diff --git a/sql/rpl_gtid.cc b/sql/rpl_gtid.cc index b40ebeafcbf7a..e4fa974d961e7 100644 --- a/sql/rpl_gtid.cc +++ b/sql/rpl_gtid.cc @@ -1898,8 +1898,6 @@ rpl_binlog_state::update_with_next_gtid(uint32 domain_id, uint32 server_id, } #endif // MYSQL_CLIENT -/* TODO: Tarun get this reviewed. changed the base class name from -rpl_gtid_base to rpl_binlog_state_base (also get reviewed the ifndef blocks )*/ /* Helper functions for update. */ int rpl_binlog_state_base::element::update_element(const rpl_gtid *gtid) From 753f9f28cb238051babfc65031023938e47daeba Mon Sep 17 00:00:00 2001 From: Tarun Wadhwa Date: Thu, 6 Aug 2026 22:40:15 +0000 Subject: [PATCH 11/14] Fixed gtid_state initialization for multiple input files --- client/mysqlbinlog.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/client/mysqlbinlog.cc b/client/mysqlbinlog.cc index bb57d3cfbe0b1..b4fda8fe02b5f 100644 --- a/client/mysqlbinlog.cc +++ b/client/mysqlbinlog.cc @@ -3946,7 +3946,9 @@ static Exit_status dump_local_log_entries(PRINT_EVENT_INFO *print_event_info, goto err; /* Initialize the GTID state tracker used to generate GTID_LIST_EVENT while converting the engine binlog to legacy binlog */ - if (opt_convert_engine_binlog && !gtid_state) { + if (opt_convert_engine_binlog) + { + delete gtid_state; gtid_state= new rpl_binlog_state_base(); gtid_state->init(); /* initialize this gtid state from the header of the innodb binlog file */ From dec6854e8add9a3d945b1c7a9837c02e546715f5 Mon Sep 17 00:00:00 2001 From: Tarun Wadhwa Date: Fri, 14 Aug 2026 00:25:42 +0000 Subject: [PATCH 12/14] Minor formatting fixes --- client/mysqlbinlog.cc | 47 ++++++---- ...sqlbinlog_convert_engine_binlog_basic.test | 5 +- ...convert_engine_binlog_gtid_list_event.test | 8 +- sql/log_event_client.cc | 5 +- sql/rpl_gtid.cc | 87 +++++++++---------- sql/sql_repl.cc | 3 +- 6 files changed, 81 insertions(+), 74 deletions(-) diff --git a/client/mysqlbinlog.cc b/client/mysqlbinlog.cc index b4fda8fe02b5f..b9aabce607a86 100644 --- a/client/mysqlbinlog.cc +++ b/client/mysqlbinlog.cc @@ -1686,8 +1686,10 @@ static void update_checksum(Log_event *ev) static bool write_format_description_event_to_legacy_binlog( FILE *outfile, Format_description_log_event *fdev) { - // temp_buf stores the raw bytes of the event and data_written is the length of those raw bytes - if(fdev->temp_buf) { + // temp_buf stores the raw bytes of the event and data_written is the length + // of those raw bytes + if (fdev->temp_buf) + { /* Update the log_file_pos */ if (update_event_end_log_pos(fdev)) return true; @@ -1695,16 +1697,12 @@ static bool write_format_description_event_to_legacy_binlog( /* recompute checksum */ update_checksum(fdev); - if (my_fwrite(outfile, (const uchar *)fdev->temp_buf, - fdev->data_written, MYF(MY_NABP))) + if (my_fwrite(outfile, (const uchar *) fdev->temp_buf, fdev->data_written, + MYF(MY_NABP))) { - error("Could not write into converted binlog file '%s'", - out_file_name); + error("Could not write into converted binlog file '%s'", out_file_name); return true; } - - fflush(outfile); - return false; } @@ -1760,7 +1758,6 @@ static bool write_format_description_event_to_legacy_binlog( out_file_name); return true; } - fflush(outfile); return false; } @@ -1807,7 +1804,6 @@ static bool write_gtid_list_event_to_legacy_binlog(FILE *outfile, out_file_name); return true; } - fflush(outfile); return false; } @@ -1877,7 +1873,6 @@ static bool write_binlog_checkpoint_event_to_legacy_binlog( out_file_name); return true; } - fflush(outfile); return false; } @@ -1919,8 +1914,13 @@ write_rotate_log_event_to_legacy_binlog(FILE *outfile, return true; } - /* we are not writing the footer because we are not supporting the checksum - in every event */ + /* write footer to output legacy binlog file */ + if (write_event_footer(outfile, do_checksum, crc)) + { + error("Could not write footer into converted binlog file '%s'", + out_file_name); + return true; + } return false; } @@ -2009,9 +2009,16 @@ static bool rotate_output_legacy_binlog(FILE **out_file, char *out_name, if (write_rotate_log_event_to_legacy_binlog(*out_file, next_out_file_name)) return true; - my_fclose(*out_file, MYF(0)); + int err= my_fclose(*out_file, MYF(0)); *out_file= NULL; + if (err) + { + error("Could not close converted binlog file '%s' during rotation", + out_name); + return true; + } + return init_output_legacy_binlog(out_file, out_name, out_name_len); } @@ -2100,7 +2107,6 @@ static Exit_status write_event_to_legacy_binlog(Log_event *ev) error("Could not write into converted binlog file '%s'", out_file_name); goto err; } - fflush(output_legacy_binlog_file); delete ev; return OK_CONTINUE; @@ -4052,7 +4058,11 @@ static Exit_status dump_local_log_entries(PRINT_EVENT_INFO *print_event_info, end: if (output_legacy_binlog_file) { - my_fclose(output_legacy_binlog_file, MYF(0)); + if (my_fclose(output_legacy_binlog_file, MYF(0))) + { + error("Could not close converted binlog file '%s'", out_file_name); + retval= ERROR_STOP; + } output_legacy_binlog_file= NULL; } if (fd >= 0) @@ -4136,7 +4146,8 @@ int main(int argc, char** argv) } if (opt_raw_mode) { - error("The --convert-engine-binlog option cannot be combined with --raw"); + error( + "The --convert-engine-binlog option cannot be combined with --raw"); die(1); } if (opt_flashback) diff --git a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.test b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.test index 815953941f8b7..5bcdc5a7764da 100644 --- a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.test +++ b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_basic.test @@ -24,10 +24,9 @@ DELETE FROM t1 WHERE a=2; REPLACE INTO t1 VALUES (3, 3); SELECT * FROM t1 ORDER BY a; -# Rotate so binlog-000000.ibb is complete on disk. Waiting for the *next* -# pre-allocated file (binlog-000002.ibb, fully pre-allocated and empty) -# guarantees both 000000 and the new active 000001 exist. FLUSH BINARY LOGS; +# Wait for binlog-000002.ibb to guarantee that all data in binlog-000000.ibb +# is available to read from disk --let $binlog_name= binlog-000002.ibb --let $binlog_size= 262144 --source include/wait_for_engine_binlog.inc diff --git a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_gtid_list_event.test b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_gtid_list_event.test index fa908dde97d69..221ce2755a634 100644 --- a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_gtid_list_event.test +++ b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_gtid_list_event.test @@ -95,8 +95,8 @@ SET @@session.gtid_domain_id= 3; SET @@session.server_id= 4; INSERT INTO t4 VALUES (1); -# this will trigger a ROTATE_EVENT and will be converted to/end of gtid_conv.000003 - +# this will trigger a FORMAT_DESCRIPTION_EVENT which on conversion will trigger +# a ROTATE_EVENT --source include/restart_mysqld.inc # start of converted gtid_conv.000004 @@ -108,7 +108,7 @@ INSERT INTO t1 VALUES (3); # end of binlog-000002.ibb which will be converted to gtid_conv.000004 FLUSH BINARY LOGS; ---let $binlog_name= binlog-000003.ibb +--let $binlog_name= binlog-000004.ibb --let $binlog_size= 262144 --source include/wait_for_engine_binlog.inc @@ -198,7 +198,7 @@ SET @@session.gtid_domain_id= 200; INSERT INTO t_large_gtid_state VALUES (200); FLUSH BINARY LOGS; ---let $binlog_name= binlog-000002.ibb +--let $binlog_name= binlog-000003.ibb --let $binlog_size= 262144 --source include/wait_for_engine_binlog.inc --enable_query_log diff --git a/sql/log_event_client.cc b/sql/log_event_client.cc index 5927730c0ad45..553d557a91209 100644 --- a/sql/log_event_client.cc +++ b/sql/log_event_client.cc @@ -2393,10 +2393,9 @@ bool Binlog_checkpoint_log_event::print(FILE *file, } /* - Constructor for Gtid_list_log_event. - Used in mysqlbinlog to generate GTID_LIST_EVENT while converting the engine binlog to legacy binlog. - TODO: Tarun get this reviewed. + Used in mysqlbinlog to generate GTID_LIST_EVENT while converting the engine + binlog to legacy binlog. */ Gtid_list_log_event::Gtid_list_log_event(rpl_binlog_state_base *gtid_set) : count(gtid_set->count_nolock()), gl_flags(0), list(0), sub_id_list(0) diff --git a/sql/rpl_gtid.cc b/sql/rpl_gtid.cc index e4fa974d961e7..dbd5afa9c1f10 100644 --- a/sql/rpl_gtid.cc +++ b/sql/rpl_gtid.cc @@ -1695,6 +1695,49 @@ rpl_binlog_state_base::find_nolock(uint32 domain_id, uint32 server_id) sizeof(server_id)); } + +/* Helper functions for update. */ +int +rpl_binlog_state_base::element::update_element(const rpl_gtid *gtid) +{ + rpl_gtid *lookup_gtid; + + /* + By far the most common case is that successive events within same + replication domain have the same server id (it changes only when + switching to a new master). So save a hash lookup in this case. + */ + if (likely(last_gtid && last_gtid->server_id == gtid->server_id)) + { + last_gtid->seq_no= gtid->seq_no; + return 0; + } + + lookup_gtid= (rpl_gtid *) + my_hash_search(&hash, (const uchar *)>id->server_id, + sizeof(gtid->server_id)); + if (lookup_gtid) + { + lookup_gtid->seq_no= gtid->seq_no; + last_gtid= lookup_gtid; + return 0; + } + + /* Allocate a new GTID and insert it. */ + lookup_gtid= (rpl_gtid *)my_malloc(PSI_INSTRUMENT_ME, sizeof(*lookup_gtid), + MYF(MY_WME)); + if (!lookup_gtid) + return 1; + memcpy(lookup_gtid, gtid, sizeof(*lookup_gtid)); + if (my_hash_insert(&hash, (const uchar *)lookup_gtid)) + { + my_free(lookup_gtid); + return 1; + } + last_gtid= lookup_gtid; + return 0; +} + #ifndef MYSQL_CLIENT /* @@ -1897,50 +1940,6 @@ rpl_binlog_state::update_with_next_gtid(uint32 domain_id, uint32 server_id, return res; } -#endif // MYSQL_CLIENT -/* Helper functions for update. */ -int -rpl_binlog_state_base::element::update_element(const rpl_gtid *gtid) -{ - rpl_gtid *lookup_gtid; - - /* - By far the most common case is that successive events within same - replication domain have the same server id (it changes only when - switching to a new master). So save a hash lookup in this case. - */ - if (likely(last_gtid && last_gtid->server_id == gtid->server_id)) - { - last_gtid->seq_no= gtid->seq_no; - return 0; - } - - lookup_gtid= (rpl_gtid *) - my_hash_search(&hash, (const uchar *)>id->server_id, - sizeof(gtid->server_id)); - if (lookup_gtid) - { - lookup_gtid->seq_no= gtid->seq_no; - last_gtid= lookup_gtid; - return 0; - } - - /* Allocate a new GTID and insert it. */ - lookup_gtid= (rpl_gtid *)my_malloc(PSI_INSTRUMENT_ME, sizeof(*lookup_gtid), - MYF(MY_WME)); - if (!lookup_gtid) - return 1; - memcpy(lookup_gtid, gtid, sizeof(*lookup_gtid)); - if (my_hash_insert(&hash, (const uchar *)lookup_gtid)) - { - my_free(lookup_gtid); - return 1; - } - last_gtid= lookup_gtid; - return 0; -} - -#ifndef MYSQL_CLIENT /* Check that a new GTID can be logged without creating an out-of-order diff --git a/sql/sql_repl.cc b/sql/sql_repl.cc index 4e80e939adc48..aa4025431b46f 100644 --- a/sql/sql_repl.cc +++ b/sql/sql_repl.cc @@ -225,8 +225,7 @@ static int fake_rotate_event(binlog_send_info *info, ulonglong position, { DBUG_ENTER("fake_rotate_event"); ulong ev_offset; - /* TODO: Tarun get this reviewed. buf should be buf[ROTATE_HEADER_LEN] */ - char buf[ROTATE_HEADER_LEN+100]; + char buf[ROTATE_HEADER_LEN]; my_bool do_checksum; int err; char* p = info->log_file_name+dirname_length(info->log_file_name); From d664f90e8f8c35ea12fe3acfb6ed94d85994f60b Mon Sep 17 00:00:00 2001 From: Tarun Wadhwa Date: Sun, 16 Aug 2026 04:39:10 +0000 Subject: [PATCH 13/14] Added tests for replication and error handling --- ...g_convert_engine_binlog_replication.result | 47 ++++++++ ...log_convert_engine_binlog_replication.test | 100 ++++++++++++++++++ ...t_engine_binlog_unsupported_options.result | 16 +++ ...ert_engine_binlog_unsupported_options.test | 37 +++++++ 4 files changed, 200 insertions(+) create mode 100644 mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_replication.result create mode 100644 mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_replication.test create mode 100644 mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_unsupported_options.result create mode 100644 mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_unsupported_options.test diff --git a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_replication.result b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_replication.result new file mode 100644 index 0000000000000..b8b58a96fbebf --- /dev/null +++ b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_replication.result @@ -0,0 +1,47 @@ +include/master-slave.inc +[connection master] +connection slave; +include/stop_slave.inc +connection master; +include/reset_master.inc +*** Generate an InnoDB-format binlog +CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1, 0), (2, 0), (3, 0); +UPDATE t1 SET b=1 WHERE a=1; +DELETE FROM t1 WHERE a=2; +REPLACE INTO t1 VALUES (3, 3); +FLUSH BINARY LOGS; +*** Convert the InnoDB-format binlog to legacy format +DROP TABLE t1; +*** Install the converted binlog on the master +include/rpl_stop_server.inc [server_number=1] +include/rpl_start_server.inc [server_number=1 parameters: --log-bin=master-bin --binlog-storage-engine= --binlog-legacy-event-pos] +*** Installed converted events have contiguous end_log_pos values +connection slave; +SET @old_parallel= @@GLOBAL.slave_parallel_threads; +SET GLOBAL slave_parallel_threads= 10; +CHANGE MASTER TO master_host='127.0.0.1', master_port=SERVER_MYPORT_1, master_user='root', master_log_file='master-bin.000001', master_log_pos=4, master_use_gtid=no; +include/start_slave.inc +*** Append new events after the converted history +connection master; +CREATE TABLE t2 (a INT PRIMARY KEY) ENGINE=InnoDB; +INSERT INTO t2 VALUES (1); +include/save_master_gtid.inc +connection slave; +include/sync_with_master_gtid.inc +*** Verify converted and newly appended events on the slave +SELECT * FROM t1 ORDER BY a; +a b +1 1 +3 3 +SELECT * FROM t2; +a +1 +include/stop_slave.inc +SET GLOBAL slave_parallel_threads= @old_parallel; +DROP TABLE t1; +CHANGE MASTER TO master_use_gtid=slave_pos; +include/start_slave.inc +connection master; +DROP TABLE t2; +include/rpl_end.inc diff --git a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_replication.test b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_replication.test new file mode 100644 index 0000000000000..25235fcadfaa2 --- /dev/null +++ b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_replication.test @@ -0,0 +1,100 @@ +# MDEV-37605 +# +# Verify that a legacy binlog generated from an InnoDB-format binlog can be +# installed on a master and replayed by a slave. Also verify that replication +# continues with events appended by the master after installing the converted +# binlog. + +--source include/have_binlog_format_row.inc +--source include/master-slave.inc +--source include/have_innodb_binlog.inc + +--connection slave +--source include/stop_slave.inc + +--connection master +--let $datadir= `SELECT @@datadir` +--source include/reset_master.inc + +--echo *** Generate an InnoDB-format binlog +CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1, 0), (2, 0), (3, 0); +UPDATE t1 SET b=1 WHERE a=1; +DELETE FROM t1 WHERE a=2; +REPLACE INTO t1 VALUES (3, 3); + +FLUSH BINARY LOGS; +# Waiting for the next preallocated file guarantees that binlog-000000.ibb is +# available on disk. +--let $binlog_name= binlog-000002.ibb +--let $binlog_size= 262144 +--source include/wait_for_engine_binlog.inc + +--echo *** Convert the InnoDB-format binlog to legacy format +--exec $MYSQL_BINLOG --convert-engine-binlog --result-file=$MYSQL_TMP_DIR/master-bin $datadir/binlog-000000.ibb + +# The converted binlog contains the history that creates t1. Remove the +# existing table before installing that history on the master. +DROP TABLE t1; + +--echo *** Install the converted binlog on the master +--let $rpl_server_number= 1 +--source include/rpl_stop_server.inc + +--remove_files_wildcard $datadir master-bin.* +--copy_file $MYSQL_TMP_DIR/master-bin.000001 $datadir/master-bin.000001 +--append_file $datadir/master-bin.index +./master-bin.000001 +EOF + +# Switch the master from InnoDB-format binlogging to the legacy binlog +--let $rpl_server_parameters= --log-bin=master-bin --binlog-storage-engine= --binlog-legacy-event-pos +--source include/rpl_start_server.inc + +# Ensure end_log_pos is populated correctly +--echo *** Installed converted events have contiguous end_log_pos values +--let $event_number= 1 +--let $end_log_pos= query_get_value(SHOW BINLOG EVENTS IN 'master-bin.000001', End_log_pos, $event_number) +--inc $event_number +--let $next_pos= query_get_value(SHOW BINLOG EVENTS IN 'master-bin.000001', Pos, $event_number) +while ($next_pos != No such row) +{ + if ($end_log_pos != $next_pos) + { + --die Event end_log_pos $end_log_pos does not match next event position $next_pos + } + --let $end_log_pos= query_get_value(SHOW BINLOG EVENTS IN 'master-bin.000001', End_log_pos, $event_number) + --inc $event_number + --let $next_pos= query_get_value(SHOW BINLOG EVENTS IN 'master-bin.000001', Pos, $event_number) +} + +--connection slave +SET @old_parallel= @@GLOBAL.slave_parallel_threads; +SET GLOBAL slave_parallel_threads= 10; +--replace_result $SERVER_MYPORT_1 SERVER_MYPORT_1 +eval CHANGE MASTER TO master_host='127.0.0.1', master_port=$SERVER_MYPORT_1, master_user='root', master_log_file='master-bin.000001', master_log_pos=4, master_use_gtid=no; +--source include/start_slave.inc + +--echo *** Append new events after the converted history +--connection master +CREATE TABLE t2 (a INT PRIMARY KEY) ENGINE=InnoDB; +INSERT INTO t2 VALUES (1); +--source include/save_master_gtid.inc + +--connection slave +--source include/sync_with_master_gtid.inc + +--echo *** Verify converted and newly appended events on the slave +SELECT * FROM t1 ORDER BY a; +SELECT * FROM t2; + +--source include/stop_slave.inc +SET GLOBAL slave_parallel_threads= @old_parallel; +DROP TABLE t1; +CHANGE MASTER TO master_use_gtid=slave_pos; +--source include/start_slave.inc + +--connection master +DROP TABLE t2; +--remove_file $MYSQL_TMP_DIR/master-bin.000001 +--source include/rpl_end.inc diff --git a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_unsupported_options.result b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_unsupported_options.result new file mode 100644 index 0000000000000..81967e01f5704 --- /dev/null +++ b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_unsupported_options.result @@ -0,0 +1,16 @@ +*** --read-from-remote-server is unsupported +ERROR: The --convert-engine-binlog option does not support --read-from-remote-server +*** --raw is unsupported +ERROR: The --convert-engine-binlog option cannot be combined with --raw +*** --flashback is unsupported +ERROR: The --convert-engine-binlog option cannot be combined with --flashback +*** --stop-datetime is unsupported +ERROR: The --convert-engine-binlog option cannot be combined with --stop-datetime +*** Numeric --stop-position is unsupported +ERROR: The --convert-engine-binlog option cannot be combined with --stop-position +*** GTID --start-position is unsupported +ERROR: The --convert-engine-binlog option does not support GTID values for --start-position or --stop-position +*** GTID --stop-position is unsupported +ERROR: The --convert-engine-binlog option does not support GTID values for --start-position or --stop-position +*** Legacy binlog input is unsupported +ERROR: The --convert-engine-binlog option requires InnoDB-engine binlog input files diff --git a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_unsupported_options.test b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_unsupported_options.test new file mode 100644 index 0000000000000..f29644df0d6d8 --- /dev/null +++ b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_unsupported_options.test @@ -0,0 +1,37 @@ +# MDEV-37605 +# +# Verify that mariadb-binlog rejects options which are incompatible with +# --convert-engine-binlog. Option validation happens before the input file is +# opened, so a placeholder input name is sufficient. + +--echo *** --read-from-remote-server is unsupported +--error 1 +--exec $MYSQL_BINLOG --convert-engine-binlog --read-from-remote-server placeholder.ibb 2>&1 + +--echo *** --raw is unsupported +--error 1 +--exec $MYSQL_BINLOG --convert-engine-binlog --raw placeholder.ibb 2>&1 + +--echo *** --flashback is unsupported +--error 1 +--exec $MYSQL_BINLOG --convert-engine-binlog --flashback placeholder.ibb 2>&1 + +--echo *** --stop-datetime is unsupported +--error 1 +--exec $MYSQL_BINLOG --convert-engine-binlog --stop-datetime="2030-01-01 00:00:00" placeholder.ibb 2>&1 + +--echo *** Numeric --stop-position is unsupported +--error 1 +--exec $MYSQL_BINLOG --convert-engine-binlog --stop-position=100 placeholder.ibb 2>&1 + +--echo *** GTID --start-position is unsupported +--error 1 +--exec $MYSQL_BINLOG --convert-engine-binlog --start-position=0-1-1 placeholder.ibb 2>&1 + +--echo *** GTID --stop-position is unsupported +--error 1 +--exec $MYSQL_BINLOG --convert-engine-binlog --stop-position=0-1-1 placeholder.ibb 2>&1 + +--echo *** Legacy binlog input is unsupported +--error 1 +--exec $MYSQL_BINLOG --convert-engine-binlog $MYSQL_TEST_DIR/std_data/mariadb-5.5-binlog.000001 2>&1 From a927632128a9e2679f9076fa0a056c52d8f1e1c0 Mon Sep 17 00:00:00 2001 From: Tarun Wadhwa Date: Thu, 20 Aug 2026 03:42:06 +0000 Subject: [PATCH 14/14] Added have_innodb_binlog.inc in unsupported options test because it was failing on embedded runs --- .../mysqlbinlog_convert_engine_binlog_unsupported_options.test | 1 + 1 file changed, 1 insertion(+) diff --git a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_unsupported_options.test b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_unsupported_options.test index f29644df0d6d8..71e086a6cca0e 100644 --- a/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_unsupported_options.test +++ b/mysql-test/suite/binlog_in_engine/mysqlbinlog_convert_engine_binlog_unsupported_options.test @@ -3,6 +3,7 @@ # Verify that mariadb-binlog rejects options which are incompatible with # --convert-engine-binlog. Option validation happens before the input file is # opened, so a placeholder input name is sufficient. +--source include/have_innodb_binlog.inc --echo *** --read-from-remote-server is unsupported --error 1