diff --git a/apps/labrinth/.sqlx/query-0f9aa21316eb3ef0ff92f0c74d5b367a5b03dd86543c7a859fd6b7f2dfef2967.json b/apps/labrinth/.sqlx/query-0f9aa21316eb3ef0ff92f0c74d5b367a5b03dd86543c7a859fd6b7f2dfef2967.json deleted file mode 100644 index 7e896a88232..00000000000 --- a/apps/labrinth/.sqlx/query-0f9aa21316eb3ef0ff92f0c74d5b367a5b03dd86543c7a859fd6b7f2dfef2967.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT v.id version_id, v.mod_id mod_id\n FROM mods m\n INNER JOIN versions v ON m.id = v.mod_id AND (cardinality($4::varchar[]) = 0 OR v.version_type = ANY($4)) AND v.status = ANY($5)\n INNER JOIN version_fields vf ON vf.field_id = 3 AND v.id = vf.version_id\n INNER JOIN loader_field_enum_values lfev ON vf.enum_value = lfev.id AND (cardinality($2::varchar[]) = 0 OR lfev.value = ANY($2::varchar[]))\n INNER JOIN loaders_versions lv ON lv.version_id = v.id\n INNER JOIN loaders l on lv.loader_id = l.id AND (cardinality($3::varchar[]) = 0 OR l.loader = ANY($3::varchar[]))\n WHERE m.id = ANY($1) AND m.status = ANY($6)\n ORDER BY v.date_published ASC\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "version_id", - "type_info": "Int8" - }, - { - "ordinal": 1, - "name": "mod_id", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Int8Array", - "VarcharArray", - "VarcharArray", - "VarcharArray", - "TextArray", - "TextArray" - ] - }, - "nullable": [ - false, - false - ] - }, - "hash": "0f9aa21316eb3ef0ff92f0c74d5b367a5b03dd86543c7a859fd6b7f2dfef2967" -} diff --git a/apps/labrinth/.sqlx/query-1c5c2944668724d4796125d8b471c736eebdebf3cd912f32175662087907aa6c.json b/apps/labrinth/.sqlx/query-1c5c2944668724d4796125d8b471c736eebdebf3cd912f32175662087907aa6c.json new file mode 100644 index 00000000000..7030defdf7b --- /dev/null +++ b/apps/labrinth/.sqlx/query-1c5c2944668724d4796125d8b471c736eebdebf3cd912f32175662087907aa6c.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH problem_groups AS (\n SELECT pag.id, pag.project_id, pag.flame_project\n FROM project_attribution_groups pag\n WHERE pag.project_id = ANY($2)\n AND (\n pag.attribution IS NULL\n OR pag.attribution->>'kind' = 'no_permission'\n OR COALESCE(\n pag.attribution->'moderation_status'->>'kind',\n 'approved'\n ) != 'approved'\n )\n AND NOT EXISTS (\n SELECT 1\n FROM attributions_exemptions ae\n WHERE ae.project_id = pag.project_id\n )\n )\n SELECT DISTINCT f.version_id AS \"version_id!: DBVersionId\", f.id AS \"file_id!: DBFileId\",\n pg.flame_project AS \"flame_project?\"\n FROM problem_groups pg\n INNER JOIN project_attribution_files paf ON paf.group_id = pg.id\n INNER JOIN override_file_sources ofs ON ofs.sha1 = paf.sha1\n INNER JOIN files f ON f.id = ofs.file_id\n INNER JOIN versions v ON v.id = f.version_id AND v.mod_id = pg.project_id\n INNER JOIN attribution_enforced_versions aev ON aev.id = f.version_id\n WHERE EXISTS (SELECT 1 FROM problem_groups)\n AND f.version_id = ANY($1)\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "version_id!: DBVersionId", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "file_id!: DBFileId", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "flame_project?", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Int8Array", + "Int8Array" + ] + }, + "nullable": [ + false, + false, + true + ] + }, + "hash": "1c5c2944668724d4796125d8b471c736eebdebf3cd912f32175662087907aa6c" +} diff --git a/apps/labrinth/.sqlx/query-4adde8c379e86b12c0b372beadd97ca77a041056b1828f897dd3c063b6cc33db.json b/apps/labrinth/.sqlx/query-4adde8c379e86b12c0b372beadd97ca77a041056b1828f897dd3c063b6cc33db.json new file mode 100644 index 00000000000..a42acfc2988 --- /dev/null +++ b/apps/labrinth/.sqlx/query-4adde8c379e86b12c0b372beadd97ca77a041056b1828f897dd3c063b6cc33db.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT q.hash AS \"hash!\", latest.id AS \"version_id!\"\n FROM JSONB_TO_RECORDSET($1::JSONB) AS q(\n hash TEXT,\n project_id BIGINT,\n loaders VARCHAR[],\n version_types VARCHAR[],\n loader_fields JSONB\n )\n CROSS JOIN LATERAL (\n SELECT v.id\n FROM versions v\n WHERE v.mod_id = q.project_id\n AND v.status = ANY($2::VARCHAR[])\n AND (q.version_types IS NULL OR v.version_type = ANY(q.version_types))\n AND (\n q.loaders IS NULL\n OR EXISTS (\n SELECT 1 FROM loaders_versions lv\n INNER JOIN loaders l ON l.id = lv.loader_id\n WHERE lv.version_id = v.id AND l.loader = ANY(q.loaders)\n )\n )\n AND NOT EXISTS (\n SELECT 1\n FROM JSONB_EACH(q.loader_fields) AS req(field, vals)\n INNER JOIN loader_fields lf ON lf.field = req.field\n WHERE EXISTS (\n SELECT 1 FROM loaders_versions lv\n INNER JOIN loader_fields_loaders lfl ON lfl.loader_id = lv.loader_id\n WHERE lv.version_id = v.id AND lfl.loader_field_id = lf.id\n )\n AND (\n lf.field_type IN ('array_integer', 'array_text', 'array_boolean', 'array_enum')\n OR (\n SELECT COUNT(*) FROM version_fields vf\n WHERE vf.version_id = v.id AND vf.field_id = lf.id\n ) = 1\n )\n AND NOT EXISTS (\n SELECT 1\n FROM version_fields vf\n LEFT JOIN loader_field_enum_values lfev ON lfev.id = vf.enum_value\n CROSS JOIN JSONB_ARRAY_ELEMENTS(req.vals) AS want(val)\n WHERE vf.version_id = v.id AND vf.field_id = lf.id\n AND CASE\n WHEN lf.field_type IN ('enum', 'array_enum')\n THEN JSONB_TYPEOF(want.val) = 'string' AND lfev.value = want.val #>> '{}'\n WHEN lf.field_type IN ('text', 'array_text')\n THEN JSONB_TYPEOF(want.val) = 'string' AND vf.string_value = want.val #>> '{}'\n WHEN lf.field_type IN ('integer', 'array_integer')\n THEN JSONB_TYPEOF(want.val) = 'number' AND vf.int_value = (want.val #>> '{}')::NUMERIC\n WHEN lf.field_type IN ('boolean', 'array_boolean')\n THEN JSONB_TYPEOF(want.val) = 'boolean' AND (vf.int_value <> 0) = (want.val #>> '{}')::BOOLEAN\n END\n )\n )\n ORDER BY v.ordering DESC NULLS FIRST, v.date_published DESC, v.id DESC\n LIMIT 1\n ) latest\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "hash!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "version_id!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Jsonb", + "VarcharArray" + ] + }, + "nullable": [ + null, + false + ] + }, + "hash": "4adde8c379e86b12c0b372beadd97ca77a041056b1828f897dd3c063b6cc33db" +} diff --git a/apps/labrinth/.sqlx/query-5cf400329120effe0eb12416a7c4558383398ce3229acff187324d8752f4eb72.json b/apps/labrinth/.sqlx/query-5cf400329120effe0eb12416a7c4558383398ce3229acff187324d8752f4eb72.json new file mode 100644 index 00000000000..57f3a130e3a --- /dev/null +++ b/apps/labrinth/.sqlx/query-5cf400329120effe0eb12416a7c4558383398ce3229acff187324d8752f4eb72.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT latest.id AS \"id!\"\n FROM mods m\n CROSS JOIN LATERAL (\n SELECT\n $7::BOOL\n OR EXISTS (\n SELECT 1 FROM team_members tm\n WHERE tm.team_id = m.team_id AND tm.user_id = $8::BIGINT\n )\n OR EXISTS (\n SELECT 1 FROM organizations o\n INNER JOIN team_members tm ON tm.team_id = o.team_id\n WHERE o.id = m.organization_id AND tm.user_id = $8::BIGINT\n ) AS full_access\n ) access\n CROSS JOIN LATERAL (\n SELECT v.id\n FROM versions v\n WHERE v.mod_id = m.id\n AND (CARDINALITY($4::VARCHAR[]) = 0 OR v.version_type = ANY($4))\n AND EXISTS (\n SELECT 1 FROM version_fields vf\n INNER JOIN loader_field_enum_values lfev ON lfev.id = vf.enum_value\n WHERE vf.version_id = v.id AND vf.field_id = 3\n AND (CARDINALITY($2::VARCHAR[]) = 0 OR lfev.value = ANY($2))\n )\n AND EXISTS (\n SELECT 1 FROM loaders_versions lv\n INNER JOIN loaders l ON l.id = lv.loader_id\n WHERE lv.version_id = v.id\n AND (CARDINALITY($3::VARCHAR[]) = 0 OR l.loader = ANY($3))\n )\n AND (\n access.full_access\n OR (\n v.status = ANY($5::VARCHAR[])\n AND NOT EXISTS (\n SELECT 1\n FROM project_attribution_groups pag\n INNER JOIN project_attribution_files paf ON paf.group_id = pag.id\n INNER JOIN override_file_sources ofs ON ofs.sha1 = paf.sha1\n INNER JOIN files f ON f.id = ofs.file_id\n INNER JOIN attribution_enforced_versions aev ON aev.id = f.version_id\n WHERE pag.project_id = v.mod_id\n AND f.version_id = v.id\n AND (\n pag.attribution IS NULL\n OR pag.attribution->>'kind' = 'no_permission'\n OR COALESCE(pag.attribution->'moderation_status'->>'kind', 'approved') != 'approved'\n )\n )\n )\n )\n ORDER BY v.date_published DESC, v.id DESC\n LIMIT 1\n ) latest\n WHERE m.id = ANY($1)\n AND (access.full_access OR m.status = ANY($6::VARCHAR[]))\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int8Array", + "VarcharArray", + "VarcharArray", + "VarcharArray", + "VarcharArray", + "VarcharArray", + "Bool", + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "5cf400329120effe0eb12416a7c4558383398ce3229acff187324d8752f4eb72" +} diff --git a/apps/labrinth/.sqlx/query-86e59f16918c0197b4e1492d3f72e191a8deb05f7753743f48f1b6647f83f544.json b/apps/labrinth/.sqlx/query-86e59f16918c0197b4e1492d3f72e191a8deb05f7753743f48f1b6647f83f544.json deleted file mode 100644 index 77323e7c62b..00000000000 --- a/apps/labrinth/.sqlx/query-86e59f16918c0197b4e1492d3f72e191a8deb05f7753743f48f1b6647f83f544.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n select distinct f.version_id as \"version_id: DBVersionId\", f.id as \"file_id: DBFileId\",\n pag.flame_project\n from files f\n inner join attribution_enforced_versions aev on aev.id = f.version_id\n inner join versions v on v.id = f.version_id\n inner join override_file_sources ofs on ofs.file_id = f.id\n inner join project_attribution_files paf on paf.sha1 = ofs.sha1\n inner join project_attribution_groups pag on pag.id = paf.group_id\n where f.version_id = ANY($1)\n and pag.project_id = v.mod_id\n and (\n pag.attribution is null\n or pag.attribution->>'kind' = 'no_permission'\n or coalesce(\n pag.attribution->'moderation_status'->>'kind',\n 'approved'\n ) != 'approved'\n )\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "version_id: DBVersionId", - "type_info": "Int8" - }, - { - "ordinal": 1, - "name": "file_id: DBFileId", - "type_info": "Int8" - }, - { - "ordinal": 2, - "name": "flame_project", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Int8Array" - ] - }, - "nullable": [ - false, - false, - true - ] - }, - "hash": "86e59f16918c0197b4e1492d3f72e191a8deb05f7753743f48f1b6647f83f544" -} diff --git a/apps/labrinth/src/auth/checks.rs b/apps/labrinth/src/auth/checks.rs index 0669ccfefd2..a5f190a31df 100644 --- a/apps/labrinth/src/auth/checks.rs +++ b/apps/labrinth/src/auth/checks.rs @@ -227,10 +227,15 @@ pub async fn filter_visible_versions( .wrap_api_err("filtering visible version ids")?; versions.retain(|x| filtered_version_ids.contains(&x.inner.id)); - let version_ids: Vec<_> = versions.iter().map(|v| v.inner.id).collect(); - let missing = get_files_missing_attribution(&**ro_pool, &version_ids) - .await - .unwrap_or_default(); + let missing = get_files_missing_attribution( + &**ro_pool, + &versions + .iter() + .map(|v| (v.inner.id, v.inner.project_id)) + .collect_vec(), + ) + .await + .unwrap_or_default(); Ok(versions .into_iter() @@ -315,10 +320,12 @@ pub async fn filter_visible_version_ids( .await .wrap_api_err("filtering enlisted version ids")?; - let version_ids: Vec<_> = versions.iter().map(|v| v.id).collect(); - let withheld_versions = get_files_missing_attribution(pool, &version_ids) - .await - .unwrap_or_default(); + let withheld_versions = get_files_missing_attribution( + pool, + &versions.iter().map(|v| (v.id, v.project_id)).collect_vec(), + ) + .await + .unwrap_or_default(); // Return versions that are not hidden, we are a mod of, or we are enlisted on the team of for version in versions { diff --git a/apps/labrinth/src/queue/file_scan.rs b/apps/labrinth/src/queue/file_scan.rs index 8ce058f8d29..e2057610e27 100644 --- a/apps/labrinth/src/queue/file_scan.rs +++ b/apps/labrinth/src/queue/file_scan.rs @@ -1209,9 +1209,15 @@ fn hash_flame_murmur32(input: Vec) -> u32 { ) } +/// Finds the files of `versions`, given as `(version, project)` pairs, that +/// bundle an override whose attribution is missing, keyed by version. +/// +/// Whether a project is exempt or has any unresolved attribution groups is +/// checked first, so files are only looked at for the projects that have +/// something to withhold. For most projects that's none of them. pub async fn get_files_missing_attribution<'a, E>( exec: E, - version_ids: &[DBVersionId], + versions: &[(DBVersionId, DBProjectId)], ) -> Result< std::collections::HashMap< DBVersionId, @@ -1221,32 +1227,48 @@ pub async fn get_files_missing_attribution<'a, E>( where E: sqlx::Executor<'a, Database = sqlx::Postgres>, { - if version_ids.is_empty() { + if versions.is_empty() { return Ok(std::collections::HashMap::new()); } let rows = sqlx::query!( r#" - select distinct f.version_id as "version_id: DBVersionId", f.id as "file_id: DBFileId", - pag.flame_project - from files f - inner join attribution_enforced_versions aev on aev.id = f.version_id - inner join versions v on v.id = f.version_id - inner join override_file_sources ofs on ofs.file_id = f.id - inner join project_attribution_files paf on paf.sha1 = ofs.sha1 - inner join project_attribution_groups pag on pag.id = paf.group_id - where f.version_id = ANY($1) - and pag.project_id = v.mod_id - and ( - pag.attribution is null - or pag.attribution->>'kind' = 'no_permission' - or coalesce( - pag.attribution->'moderation_status'->>'kind', - 'approved' - ) != 'approved' - ) + WITH problem_groups AS ( + SELECT pag.id, pag.project_id, pag.flame_project + FROM project_attribution_groups pag + WHERE pag.project_id = ANY($2) + AND ( + pag.attribution IS NULL + OR pag.attribution->>'kind' = 'no_permission' + OR COALESCE( + pag.attribution->'moderation_status'->>'kind', + 'approved' + ) != 'approved' + ) + AND NOT EXISTS ( + SELECT 1 + FROM attributions_exemptions ae + WHERE ae.project_id = pag.project_id + ) + ) + SELECT DISTINCT f.version_id AS "version_id!: DBVersionId", f.id AS "file_id!: DBFileId", + pg.flame_project AS "flame_project?" + FROM problem_groups pg + INNER JOIN project_attribution_files paf ON paf.group_id = pg.id + INNER JOIN override_file_sources ofs ON ofs.sha1 = paf.sha1 + INNER JOIN files f ON f.id = ofs.file_id + INNER JOIN versions v ON v.id = f.version_id AND v.mod_id = pg.project_id + INNER JOIN attribution_enforced_versions aev ON aev.id = f.version_id + WHERE EXISTS (SELECT 1 FROM problem_groups) + AND f.version_id = ANY($1) "#, - &version_ids.iter().map(|v| v.0).collect::>(), + &versions.iter().map(|(v, _)| v.0).collect::>(), + &versions + .iter() + .map(|(_, p)| p.0) + .collect::>() + .into_iter() + .collect::>(), ) .fetch_all(exec) .await diff --git a/apps/labrinth/src/routes/v2/version_file.rs b/apps/labrinth/src/routes/v2/version_file.rs index 3aeb40fbf9d..4e88718289a 100644 --- a/apps/labrinth/src/routes/v2/version_file.rs +++ b/apps/labrinth/src/routes/v2/version_file.rs @@ -545,7 +545,7 @@ pub struct ManyFileUpdateData { #[post("/update_individual")] pub async fn update_individual_files( req: HttpRequest, - pool: web::Data, + pool: web::Data, redis: web::Data, update_data: web::Json, session_queue: web::Data, diff --git a/apps/labrinth/src/routes/v3/version_file.rs b/apps/labrinth/src/routes/v3/version_file.rs index ec04257bcdb..3ebd497e40e 100644 --- a/apps/labrinth/src/routes/v3/version_file.rs +++ b/apps/labrinth/src/routes/v3/version_file.rs @@ -16,8 +16,6 @@ use crate::util::error::ApiContext as _; use crate::util::error::Context; use crate::{database, models}; use actix_web::{HttpRequest, HttpResponse, delete, get, post, web}; -use dashmap::DashMap; -use futures::TryStreamExt; use itertools::Itertools; use serde::{Deserialize, Serialize}; use std::collections::HashMap; @@ -221,55 +219,33 @@ pub async fn get_update_from_hash( hash_query.algorithm.clone().unwrap_or_else(|| { default_algorithm_from_hashes(std::slice::from_ref(&hash)) }), - hash, + hash.clone(), hash_query.version_id.map(|x| x.into()), &***pool, &redis, ) .await .wrap_internal_err("querying database for `get_update_from_hash`")? - && let Some(project) = database::models::DBProject::get_id( - file.project_id, - &***pool, - &redis, - ) - .await - .wrap_internal_err("fetching project for version file")? { - let mut versions = database::models::DBVersion::get_many( - &project.versions, - &***pool, - &redis, + let latest = get_latest_matching_versions( + &[LatestVersionFilter { + hash: &hash, + project_id: file.project_id.0, + loaders: update_data.loaders.as_deref(), + version_types: update_data.version_types.as_deref(), + loader_fields: update_data.loader_fields.as_ref(), + }], + &pool, ) .await - .wrap_internal_err("fetching versions from database")? - .into_iter() - .filter(|x| { - let mut bool = true; - if let Some(version_types) = &update_data.version_types { - bool &= version_types - .iter() - .any(|y| y.as_str() == x.inner.version_type); - } - if let Some(loaders) = &update_data.loaders { - bool &= x.loaders.iter().any(|y| loaders.contains(y)); - } - if let Some(loader_fields) = &update_data.loader_fields { - for (key, values) in loader_fields { - bool &= if let Some(x_vf) = - x.version_fields.iter().find(|y| y.field_name == *key) - { - values.iter().any(|v| x_vf.value.contains_json_value(v)) - } else { - true - }; - } - } - bool - }) - .sorted(); + .wrap_api_err("fetching latest matching version")?; - if let Some(first) = versions.next_back() { + if let Some(version_id) = latest.into_values().next() + && let Some(first) = + database::models::DBVersion::get(version_id, &***pool, &redis) + .await + .wrap_internal_err("fetching version from database")? + { if !is_visible_version(&first.inner, &user_option, &pool, &redis) .await .wrap_api_err("checking version visibility")? @@ -573,6 +549,13 @@ pub async fn update_files( Ok(web::Json(resp)) } +/// Finds, for each project, the newest version matching the filters that the +/// user can see. +/// +/// Visibility mirrors `filter_visible_version_ids`: moderators and members of +/// the project's team or organization see every version, everyone else only +/// sees non-hidden versions of non-hidden projects that aren't withheld for +/// missing attribution (see `get_files_missing_attribution`). async fn update_files_internal( req: HttpRequest, pool: web::Data, @@ -605,71 +588,104 @@ async fn update_files_internal( .wrap_internal_err("updating versions in database")?; // TODO: de-hardcode this and actually use version fields system - let update_version_ids = sqlx::query!( - " - SELECT v.id version_id, v.mod_id mod_id + let latest_version_ids = sqlx::query_scalar!( + r#" + SELECT latest.id AS "id!" FROM mods m - INNER JOIN versions v ON m.id = v.mod_id AND (cardinality($4::varchar[]) = 0 OR v.version_type = ANY($4)) AND v.status = ANY($5) - INNER JOIN version_fields vf ON vf.field_id = 3 AND v.id = vf.version_id - INNER JOIN loader_field_enum_values lfev ON vf.enum_value = lfev.id AND (cardinality($2::varchar[]) = 0 OR lfev.value = ANY($2::varchar[])) - INNER JOIN loaders_versions lv ON lv.version_id = v.id - INNER JOIN loaders l on lv.loader_id = l.id AND (cardinality($3::varchar[]) = 0 OR l.loader = ANY($3::varchar[])) - WHERE m.id = ANY($1) AND m.status = ANY($6) - ORDER BY v.date_published ASC - ", + CROSS JOIN LATERAL ( + SELECT + $7::BOOL + OR EXISTS ( + SELECT 1 FROM team_members tm + WHERE tm.team_id = m.team_id AND tm.user_id = $8::BIGINT + ) + OR EXISTS ( + SELECT 1 FROM organizations o + INNER JOIN team_members tm ON tm.team_id = o.team_id + WHERE o.id = m.organization_id AND tm.user_id = $8::BIGINT + ) AS full_access + ) access + CROSS JOIN LATERAL ( + SELECT v.id + FROM versions v + WHERE v.mod_id = m.id + AND (CARDINALITY($4::VARCHAR[]) = 0 OR v.version_type = ANY($4)) + AND EXISTS ( + SELECT 1 FROM version_fields vf + INNER JOIN loader_field_enum_values lfev ON lfev.id = vf.enum_value + WHERE vf.version_id = v.id AND vf.field_id = 3 + AND (CARDINALITY($2::VARCHAR[]) = 0 OR lfev.value = ANY($2)) + ) + AND EXISTS ( + SELECT 1 FROM loaders_versions lv + INNER JOIN loaders l ON l.id = lv.loader_id + WHERE lv.version_id = v.id + AND (CARDINALITY($3::VARCHAR[]) = 0 OR l.loader = ANY($3)) + ) + AND ( + access.full_access + OR ( + v.status = ANY($5::VARCHAR[]) + AND NOT EXISTS ( + SELECT 1 + FROM project_attribution_groups pag + INNER JOIN project_attribution_files paf ON paf.group_id = pag.id + INNER JOIN override_file_sources ofs ON ofs.sha1 = paf.sha1 + INNER JOIN files f ON f.id = ofs.file_id + INNER JOIN attribution_enforced_versions aev ON aev.id = f.version_id + WHERE pag.project_id = v.mod_id + AND f.version_id = v.id + AND ( + pag.attribution IS NULL + OR pag.attribution->>'kind' = 'no_permission' + OR COALESCE(pag.attribution->'moderation_status'->>'kind', 'approved') != 'approved' + ) + ) + ) + ) + ORDER BY v.date_published DESC, v.id DESC + LIMIT 1 + ) latest + WHERE m.id = ANY($1) + AND (access.full_access OR m.status = ANY($6::VARCHAR[])) + "#, &files.iter().map(|x| x.project_id.0).collect::>(), &update_data.game_versions.clone().unwrap_or_default(), &update_data.loaders.clone().unwrap_or_default(), - &update_data.version_types.clone().unwrap_or_default().iter().map(|x| x.to_string()).collect::>(), - &*VersionStatus::iterator() - .map(|x| x.to_string()) - .collect::>(), - &*ProjectStatus::iterator() - .map(|x| x.to_string()) - .collect::>(), - ) - .fetch(&***pool) - .try_fold(DashMap::new(), |acc : DashMap<_,Vec>, m| { - acc.entry(database::models::DBProjectId(m.mod_id)) - .or_default() - .push(database::models::DBVersionId(m.version_id)); - async move { Ok(acc) } - }) - .await - .wrap_internal_err("fetching project version IDs from database")?; - - let candidate_versions = database::models::DBVersion::get_many( - &update_version_ids + &update_data + .version_types + .clone() + .unwrap_or_default() .iter() - .flat_map(|x| x.value().clone()) + .map(|x| x.to_string()) .collect::>(), - &***pool, - &redis, - ) - .await - .wrap_internal_err("updating versions in database")?; - let visible_version_ids = filter_visible_version_ids( - candidate_versions.iter().map(|x| &x.inner).collect(), - &user_option, - &pool, - &redis, + &VersionStatus::iterator() + .filter(|x| !x.is_hidden()) + .map(|x| x.to_string()) + .collect::>(), + &ProjectStatus::iterator() + .filter(|x| !x.is_hidden()) + .map(|x| x.to_string()) + .collect::>(), + user_option.as_ref().is_some_and(|x| x.role.is_mod()), + user_option + .as_ref() + .map(|x| database::models::DBUserId::from(x.id).0), ) + .fetch_all(&***pool) .await - .wrap_api_err("filtering visible update versions")?; + .wrap_internal_err("fetching latest visible update versions")?; + let versions = database::models::DBVersion::get_many( - &update_version_ids + &latest_version_ids .into_iter() - .filter_map(|x| { - x.1.into_iter() - .rev() - .find(|id| visible_version_ids.contains(id)) - }) + .map(database::models::DBVersionId) .collect::>(), &***pool, &redis, ) .await - .wrap_internal_err("fetching latest visible update versions")?; + .wrap_internal_err("fetching update versions")?; let mut response = HashMap::>::new(); for file in files { @@ -719,7 +735,7 @@ pub struct ManyFileUpdateData { #[post("/version_files/update_individual")] pub async fn update_individual_files_route( req: HttpRequest, - pool: web::Data, + pool: web::Data, redis: web::Data, update_data: web::Json, session_queue: web::Data, @@ -729,14 +745,14 @@ pub async fn update_individual_files_route( pub async fn update_individual_files( req: HttpRequest, - pool: web::Data, + pool: web::Data, redis: web::Data, update_data: web::Json, session_queue: web::Data, ) -> Result { let user_option = get_user_from_headers( &req, - &**pool, + &***pool, &redis, &session_queue, Scopes::VERSION_READ, @@ -761,94 +777,162 @@ pub async fn update_individual_files( .iter() .map(|x| x.hash.clone()) .collect::>(), - &**pool, + &***pool, &redis, ) .await .wrap_internal_err("updating versions in database")?; - let projects = database::models::DBProject::get_many_ids( - &files.iter().map(|x| x.project_id).collect::>(), - &**pool, + let filters = files + .iter() + .filter_map(|file| { + let hash = file.hashes.get(&algorithm)?; + let query_file = + update_data.hashes.iter().find(|x| &x.hash == hash)?; + Some(LatestVersionFilter { + hash, + project_id: file.project_id.0, + loaders: query_file.loaders.as_deref(), + version_types: query_file.version_types.as_deref(), + loader_fields: query_file.loader_fields.as_ref(), + }) + }) + .collect::>(); + let latest = get_latest_matching_versions(&filters, &pool) + .await + .wrap_api_err("fetching latest matching versions")?; + + let versions = database::models::DBVersion::get_many( + &latest.values().copied().unique().collect::>(), + &***pool, &redis, ) .await - .wrap_internal_err("fetching projects for version files")?; - let all_versions = database::models::DBVersion::get_many( - &projects - .iter() - .flat_map(|x| x.versions.clone()) - .collect::>(), - &**pool, + .wrap_internal_err("fetching versions from database")?; + let visible_version_ids = filter_visible_version_ids( + versions.iter().map(|x| &x.inner).collect(), + &user_option, + &pool, &redis, ) .await - .wrap_internal_err("fetching versions from database")?; + .wrap_api_err("filtering visible update versions")?; - let mut response = HashMap::new(); + let response = latest + .into_iter() + .filter(|(_, version_id)| visible_version_ids.contains(version_id)) + .filter_map(|(hash, version_id)| { + let version = versions.iter().find(|x| x.inner.id == version_id)?; + Some((hash, models::projects::Version::from(version.clone()))) + }) + .collect::>(); - for project in projects { - for file in files.iter().filter(|x| x.project_id == project.inner.id) { - if let Some(hash) = file.hashes.get(&algorithm) - && let Some(query_file) = - update_data.hashes.iter().find(|x| &x.hash == hash) - { - let version = all_versions - .iter() - .filter(|x| x.inner.project_id == file.project_id) - .filter(|x| { - let mut bool = true; - - if let Some(version_types) = &query_file.version_types { - bool &= version_types - .iter() - .any(|y| y.as_str() == x.inner.version_type); - } - if let Some(loaders) = &query_file.loaders { - bool &= - x.loaders.iter().any(|y| loaders.contains(y)); - } - - if let Some(loader_fields) = &query_file.loader_fields { - for (key, values) in loader_fields { - bool &= if let Some(x_vf) = x - .version_fields - .iter() - .find(|y| y.field_name == *key) - { - values.iter().any(|v| { - x_vf.value.contains_json_value(v) - }) - } else { - true - }; - } - } - bool - }) - .sorted() - .next_back(); - - if let Some(version) = version - && is_visible_version( - &version.inner, - &user_option, - &pool, - &redis, - ) - .await - .wrap_api_err("checking version visibility")? - { - response.insert( - hash.clone(), - models::projects::Version::from(version.clone()), - ); - } - } - } + Ok(HttpResponse::Ok().json(response)) +} + +/// Per-hash filters for [`get_latest_matching_versions`], decoded by Postgres +/// with `jsonb_to_recordset`. +#[derive(Serialize)] +struct LatestVersionFilter<'a> { + hash: &'a str, + project_id: i64, + loaders: Option<&'a [String]>, + version_types: Option<&'a [VersionType]>, + loader_fields: Option<&'a HashMap>>, +} + +/// Finds, for each filter, the newest listed version of its project matching +/// its loaders, version types and loader fields, keyed by the filter's hash. +/// +/// Versions are ordered like `impl Ord for DBVersion`, and visibility is left +/// to the caller. A loader field only rules out versions that have it, using +/// the same rules as `VersionField::from_query_json`: the field must be linked +/// to one of the version's loaders, and a non-array field must have exactly one +/// value. +async fn get_latest_matching_versions( + filters: &[LatestVersionFilter<'_>], + pool: &PgPool, +) -> Result, ApiError> { + if filters.is_empty() { + return Ok(HashMap::new()); } - Ok(HttpResponse::Ok().json(response)) + let rows = sqlx::query!( + r#" + SELECT q.hash AS "hash!", latest.id AS "version_id!" + FROM JSONB_TO_RECORDSET($1::JSONB) AS q( + hash TEXT, + project_id BIGINT, + loaders VARCHAR[], + version_types VARCHAR[], + loader_fields JSONB + ) + CROSS JOIN LATERAL ( + SELECT v.id + FROM versions v + WHERE v.mod_id = q.project_id + AND v.status = ANY($2::VARCHAR[]) + AND (q.version_types IS NULL OR v.version_type = ANY(q.version_types)) + AND ( + q.loaders IS NULL + OR EXISTS ( + SELECT 1 FROM loaders_versions lv + INNER JOIN loaders l ON l.id = lv.loader_id + WHERE lv.version_id = v.id AND l.loader = ANY(q.loaders) + ) + ) + AND NOT EXISTS ( + SELECT 1 + FROM JSONB_EACH(q.loader_fields) AS req(field, vals) + INNER JOIN loader_fields lf ON lf.field = req.field + WHERE EXISTS ( + SELECT 1 FROM loaders_versions lv + INNER JOIN loader_fields_loaders lfl ON lfl.loader_id = lv.loader_id + WHERE lv.version_id = v.id AND lfl.loader_field_id = lf.id + ) + AND ( + lf.field_type IN ('array_integer', 'array_text', 'array_boolean', 'array_enum') + OR ( + SELECT COUNT(*) FROM version_fields vf + WHERE vf.version_id = v.id AND vf.field_id = lf.id + ) = 1 + ) + AND NOT EXISTS ( + SELECT 1 + FROM version_fields vf + LEFT JOIN loader_field_enum_values lfev ON lfev.id = vf.enum_value + CROSS JOIN JSONB_ARRAY_ELEMENTS(req.vals) AS want(val) + WHERE vf.version_id = v.id AND vf.field_id = lf.id + AND CASE + WHEN lf.field_type IN ('enum', 'array_enum') + THEN JSONB_TYPEOF(want.val) = 'string' AND lfev.value = want.val #>> '{}' + WHEN lf.field_type IN ('text', 'array_text') + THEN JSONB_TYPEOF(want.val) = 'string' AND vf.string_value = want.val #>> '{}' + WHEN lf.field_type IN ('integer', 'array_integer') + THEN JSONB_TYPEOF(want.val) = 'number' AND vf.int_value = (want.val #>> '{}')::NUMERIC + WHEN lf.field_type IN ('boolean', 'array_boolean') + THEN JSONB_TYPEOF(want.val) = 'boolean' AND (vf.int_value <> 0) = (want.val #>> '{}')::BOOLEAN + END + ) + ) + ORDER BY v.ordering DESC NULLS FIRST, v.date_published DESC, v.id DESC + LIMIT 1 + ) latest + "#, + sqlx::types::Json(filters) as _, + &VersionStatus::iterator() + .filter(|x| x.is_listed()) + .map(|x| x.to_string()) + .collect::>(), + ) + .fetch_all(pool) + .await + .wrap_internal_err("fetching latest matching versions")?; + + Ok(rows + .into_iter() + .map(|row| (row.hash, database::models::DBVersionId(row.version_id))) + .collect()) } // under /api/v1/version_file/{hash} diff --git a/apps/labrinth/src/routes/v3/versions.rs b/apps/labrinth/src/routes/v3/versions.rs index ff5022fa4c5..06007dd0b17 100644 --- a/apps/labrinth/src/routes/v3/versions.rs +++ b/apps/labrinth/src/routes/v3/versions.rs @@ -123,11 +123,14 @@ pub async fn version_project_get_helper( .wrap_api_err("checking version visibility")? { let version_id = version.inner.id; + let project_id = version.inner.project_id; let mut v = models::projects::Version::from(version); - let missing = - get_files_missing_attribution(&***ro_pool, &[version_id]) - .await - .unwrap_or_default(); + let missing = get_files_missing_attribution( + &***ro_pool, + &[(version_id, project_id)], + ) + .await + .unwrap_or_default(); v.files_missing_attribution = missing .get(&version_id) .map(|entries| { @@ -310,10 +313,14 @@ pub async fn version_get_helper( .wrap_api_err("checking version visibility")? { let version_id = data.inner.id; + let project_id = data.inner.project_id; let mut version = models::projects::Version::from(data); - let missing = get_files_missing_attribution(&***ro_pool, &[version_id]) - .await - .unwrap_or_default(); + let missing = get_files_missing_attribution( + &***ro_pool, + &[(version_id, project_id)], + ) + .await + .unwrap_or_default(); version.files_missing_attribution = missing .get(&version_id) .map(|entries| {