[2.0 Breaking] Add --include-private and default private tables to not generate (#4241)

# Description of Changes
Updated the codegen table/function iteration functions to take in a
parameter to check visibility in all locations for the supported
languages.
- Updated the util.rs functions for iterating tables/functions to check
for a CodegenVisibility enum (IncludePrivate, or OnlyPublic)
- Added a new CodegenOptions struct to pass around the CodegenVisibility
and future flags, defaulted visibility to OnlyPublic
- Updated the CLI to return a list of all private tables not included
(added a TODO to check the --include-private opt):
```bash
Optimising module with wasm-opt...
Build finished successfully.
Skipping private tables during codegen: secret_note, secret_order, secret_person.
Generate finished successfully.
```

# API and ABI breaking changes

Technically API breaking as the private tables will no longer be
available. (GitHub labels are not working at the moment)

# Expected complexity level and risk

1 - Simple change the testing took longer

# Testing

Turns out when you remove private tables you invalidate most of the
module_bindings across the system!

- [x] Rust test SDK for all languages
- [x] C# SDK tests
- [x] C# dotnet tests
- [x] Updated and checked snap files
- [x] Updated Blackholio (Unreal + Unity) module_bindings and tested
- [x] Ran Unreal SDK tests

---------

Signed-off-by: Jason Larabie <jason@clockworklabs.io>

Author: JasonAtClockwork

.gitignore

@@ -234,6 +234,10 @@ new.json
 # C++ build data
 modules/benchmarks-cpp/build/
 modules/module-test-cpp/build/
+modules/sdk-test-cpp/build/
+modules/sdk-test-connect-disconnect-cpp/build/
+modules/sdk-test-procedure-cpp/build/
+modules/sdk-test-view-cpp/build/
 
 # Symlinked output from `nix build`
 result

crates/cli/src/subcommands/generate.rs

@@ -3,9 +3,12 @@
 use anyhow::Context;
 use clap::parser::ValueSource;
 use clap::Arg;
-use clap::ArgAction::Set;
+use clap::ArgAction::{Set, SetTrue};
 use fs_err as fs;
-use spacetimedb_codegen::{generate, Csharp, Lang, OutputFile, Rust, TypeScript, UnrealCpp, AUTO_GENERATED_PREFIX};
+use spacetimedb_codegen::{
+    generate, private_table_names, CodegenOptions, CodegenVisibility, Csharp, Lang, OutputFile, Rust, TypeScript,
+    UnrealCpp, AUTO_GENERATED_PREFIX,
+};
 use spacetimedb_lib::de::serde::DeserializeWrapper;
 use spacetimedb_lib::{sats, RawModuleDef};
 use spacetimedb_schema;
@@ -25,7 +28,7 @@ use std::io::Read;
 pub fn cli() -> clap::Command {
     clap::Command::new("generate")
         .about("Generate client files for a spacetime module.")
-        .override_usage("spacetime generate --lang <LANG> --out-dir <DIR> [--project-path <DIR> | --bin-path <PATH> | --module-name <MODULE_NAME> | --uproject-dir <DIR>]")
+        .override_usage("spacetime generate --lang <LANG> --out-dir <DIR> [--project-path <DIR> | --bin-path <PATH> | --module-name <MODULE_NAME> | --uproject-dir <DIR> | --include-private]")
         .arg(
             Arg::new("wasm_file")
                 .value_parser(clap::value_parser!(PathBuf))
@@ -109,6 +112,13 @@ pub fn cli() -> clap::Command {
                 .default_value("")
                 .help("Options to pass to the build command, for example --build-options='--lint-dir='"),
         )
+        .arg(
+            Arg::new("include_private")
+                .long("include-private")
+                .action(SetTrue)
+                .default_value("false")
+                .help("Include private tables and functions in generated code (types are always included)."),
+        )
         .arg(common_args::yes())
         .after_help("Run `spacetime help publish` for more detailed information.")
         .group(
@@ -137,6 +147,7 @@ pub async fn exec_ex(
     let module_name = args.get_one::<String>("module_name");
     let force = args.get_flag("force");
     let build_options = args.get_one::<String>("build_options").unwrap();
+    let include_private = args.get_flag("include_private");
 
     if args.value_source("namespace") == Some(ValueSource::CommandLine) && lang != Language::Csharp {
         return Err(anyhow::anyhow!("--namespace is only supported with --lang csharp"));
@@ -171,6 +182,16 @@ pub async fn exec_ex(
         extract_descriptions(&path).context("could not extract schema")?
     };
 
+    let private_tables = private_table_names(&module);
+    if !private_tables.is_empty() && !include_private {
+        println!("Skipping private tables during codegen: {}.", private_tables.join(", "));
+    }
+
+    let mut options = CodegenOptions::default();
+    if include_private {
+        options.visibility = CodegenVisibility::IncludePrivate;
+    }
+
     fs::create_dir_all(out_dir)?;
 
     let mut paths = BTreeSet::new();
@@ -193,7 +214,7 @@ pub async fn exec_ex(
         Language::TypeScript => &TypeScript,
     };
 
-    for OutputFile { filename, code } in generate(&module, gen_lang) {
+    for OutputFile { filename, code } in generate(&module, gen_lang, &options) {
         let fname = Path::new(&filename);
         // If a generator asks for a file in a subdirectory, create the subdirectory first.
         if let Some(parent) = fname.parent().filter(|p| !p.as_os_str().is_empty()) {

crates/codegen/examples/regen-cpp-moduledef.rs

@@ -2,7 +2,7 @@
 //! Run `cargo run --example regen-cpp-moduledef` to update C++ bindings whenever the module definition changes.
 
 use fs_err as fs;
-use spacetimedb_codegen::{cpp, generate, OutputFile};
+use spacetimedb_codegen::{cpp, generate, CodegenOptions, OutputFile};
 use spacetimedb_lib::db::raw_def::v9::{RawModuleDefV9, RawModuleDefV9Builder};
 use spacetimedb_schema::def::ModuleDef;
 use std::path::Path;
@@ -33,6 +33,7 @@ fn main() -> anyhow::Result<()> {
         &cpp::Cpp {
             namespace: "SpacetimeDB::Internal",
         },
+        &CodegenOptions::default(),
     )
     .into_iter()
     .try_for_each(|OutputFile { filename, code }| {

crates/codegen/examples/regen-csharp-moduledef.rs

@@ -3,7 +3,7 @@
 
 use fs_err as fs;
 use regex::Regex;
-use spacetimedb_codegen::{csharp, generate, OutputFile};
+use spacetimedb_codegen::{csharp, generate, CodegenOptions, OutputFile};
 use spacetimedb_lib::{RawModuleDef, RawModuleDefV8};
 use spacetimedb_schema::def::ModuleDef;
 use std::path::Path;
@@ -37,6 +37,7 @@ fn main() -> anyhow::Result<()> {
         &csharp::Csharp {
             namespace: "SpacetimeDB.Internal",
         },
+        &CodegenOptions::default(),
     )
     .into_iter()
     .try_for_each(|OutputFile { filename, code }| {

crates/codegen/examples/regen-typescript-moduledef.rs

@@ -5,7 +5,7 @@
 
 use fs_err as fs;
 use regex::Regex;
-use spacetimedb_codegen::{generate, typescript, OutputFile};
+use spacetimedb_codegen::{generate, typescript, CodegenOptions, OutputFile};
 use spacetimedb_lib::db::raw_def::v9::ViewResultHeader;
 use spacetimedb_lib::{RawModuleDef, RawModuleDefV8};
 use spacetimedb_schema::def::ModuleDef;
@@ -38,7 +38,7 @@ fn main() -> anyhow::Result<()> {
     fs::create_dir(dir)?;
 
     let module: ModuleDef = module.try_into()?;
-    generate(&module, &typescript::TypeScript)
+    generate(&module, &typescript::TypeScript, &CodegenOptions::default())
         .into_iter()
         .try_for_each(|OutputFile { filename, code }| {
             // Skip the index.ts since we don't need it.

crates/codegen/src/cpp.rs

@@ -1,6 +1,7 @@
 //! Minimal C++ code generation for SpacetimeDB module definitions.
 //! Generates only schema definitions - framework provides all functionality.
 
+use crate::CodegenOptions;
 use crate::Lang;
 use crate::OutputFile;
 use spacetimedb_lib::sats::layout::PrimitiveType;
@@ -557,7 +558,7 @@ impl Lang for Cpp<'_> {
         }
     }
 
-    fn generate_global_files(&self, _module: &ModuleDef) -> Vec<OutputFile> {
+    fn generate_global_files(&self, _module: &ModuleDef, _options: &CodegenOptions) -> Vec<OutputFile> {
         vec![]
     }
 }

crates/codegen/src/csharp.rs

@@ -11,7 +11,7 @@ use crate::util::{
     collect_case, is_reducer_invokable, iter_indexes, iter_reducers, iter_table_names_and_types,
     print_auto_generated_file_comment, print_auto_generated_version_comment, type_ref_name,
 };
-use crate::{indent_scope, OutputFile};
+use crate::{indent_scope, CodegenOptions, OutputFile};
 use convert_case::{Case, Casing};
 use spacetimedb_lib::sats::layout::PrimitiveType;
 use spacetimedb_primitives::ColId;
@@ -1010,7 +1010,7 @@ impl Lang for Csharp<'_> {
         }
     }
 
-    fn generate_global_files(&self, module: &ModuleDef) -> Vec<OutputFile> {
+    fn generate_global_files(&self, module: &ModuleDef, options: &CodegenOptions) -> Vec<OutputFile> {
         let mut output = CsharpAutogen::new(
             self.namespace,
             &[
@@ -1048,7 +1048,7 @@ impl Lang for Csharp<'_> {
         indented_block(&mut output, |output| {
             writeln!(output, "public RemoteTables(DbConnection conn)");
             indented_block(output, |output| {
-                for (table_name, _) in iter_table_names_and_types(module) {
+                for (table_name, _) in iter_table_names_and_types(module, options.visibility) {
                     writeln!(
                         output,
                         "AddTable({} = new(conn));",
@@ -1071,7 +1071,7 @@ impl Lang for Csharp<'_> {
 
         writeln!(output, "public sealed class From");
         indented_block(&mut output, |output| {
-            for (table_name, product_type_ref) in iter_table_names_and_types(module) {
+            for (table_name, product_type_ref) in iter_table_names_and_types(module, options.visibility) {
                 let method_name = table_name.deref().to_case(Case::Pascal);
                 let row_type = type_ref_name(module, product_type_ref);
                 let table_name_lit = format!("{:?}", table_name.deref());
@@ -1176,7 +1176,7 @@ impl Lang for Csharp<'_> {
                 writeln!(output, "return update.ReducerCall.ReducerName switch {{");
                 {
                     indent_scope!(output);
-                    for reducer in iter_reducers(module) {
+                    for reducer in iter_reducers(module, options.visibility) {
                         let reducer_str_name = &reducer.name;
                         let reducer_name = reducer.name.deref().to_case(Case::Pascal);
                         writeln!(
@@ -1241,7 +1241,9 @@ impl Lang for Csharp<'_> {
                 writeln!(output, "return reducer switch {{");
                 {
                     indent_scope!(output);
-                    for reducer_name in iter_reducers(module).map(|r| r.name.deref().to_case(Case::Pascal)) {
+                    for reducer_name in
+                        iter_reducers(module, options.visibility).map(|r| r.name.deref().to_case(Case::Pascal))
+                    {
                         writeln!(
                             output,
                             "Reducer.{reducer_name} args => Reducers.Invoke{reducer_name}(eventContext, args),"

crates/codegen/src/lib.rs

@@ -12,16 +12,32 @@ pub use self::csharp::Csharp;
 pub use self::rust::Rust;
 pub use self::typescript::TypeScript;
 pub use self::unrealcpp::UnrealCpp;
+pub use util::private_table_names;
+pub use util::CodegenVisibility;
 pub use util::AUTO_GENERATED_PREFIX;
 
-pub fn generate(module: &ModuleDef, lang: &dyn Lang) -> Vec<OutputFile> {
+#[derive(Clone, Copy, Debug)]
+pub struct CodegenOptions {
+    pub visibility: CodegenVisibility,
+}
+
+impl Default for CodegenOptions {
+    fn default() -> Self {
+        Self {
+            visibility: CodegenVisibility::OnlyPublic,
+        }
+    }
+}
+
+pub fn generate(module: &ModuleDef, lang: &dyn Lang, options: &CodegenOptions) -> Vec<OutputFile> {
     itertools::chain!(
-        module.tables().map(|tbl| lang.generate_table_file(module, tbl)),
+        util::iter_tables(module, options.visibility).map(|tbl| lang.generate_table_file(module, tbl)),
         module.views().map(|view| lang.generate_view_file(module, view)),
         module.types().flat_map(|typ| lang.generate_type_files(module, typ)),
-        util::iter_reducers(module).map(|reducer| lang.generate_reducer_file(module, reducer)),
-        util::iter_procedures(module).map(|procedure| lang.generate_procedure_file(module, procedure)),
-        lang.generate_global_files(module),
+        util::iter_reducers(module, options.visibility).map(|reducer| lang.generate_reducer_file(module, reducer)),
+        util::iter_procedures(module, options.visibility)
+            .map(|procedure| lang.generate_procedure_file(module, procedure)),
+        lang.generate_global_files(module, options),
     )
     .collect()
 }
@@ -36,7 +52,7 @@ pub trait Lang {
     fn generate_type_files(&self, module: &ModuleDef, typ: &TypeDef) -> Vec<OutputFile>;
     fn generate_reducer_file(&self, module: &ModuleDef, reducer: &ReducerDef) -> OutputFile;
     fn generate_procedure_file(&self, module: &ModuleDef, procedure: &ProcedureDef) -> OutputFile;
-    fn generate_global_files(&self, module: &ModuleDef) -> Vec<OutputFile>;
+    fn generate_global_files(&self, module: &ModuleDef, options: &CodegenOptions) -> Vec<OutputFile>;
 
     fn generate_table_file(&self, module: &ModuleDef, tbl: &TableDef) -> OutputFile {
         let schema = TableSchema::from_module_def(module, tbl, (), 0.into())