diff --git a/apps/importer/src/shopee/ai-review-analyzer.ts b/apps/importer/src/shopee/ai-review-analyzer.ts index 7df0d73..c964adb 100644 --- a/apps/importer/src/shopee/ai-review-analyzer.ts +++ b/apps/importer/src/shopee/ai-review-analyzer.ts @@ -1,6 +1,7 @@ import crypto from 'node:crypto'; import type { SupabaseClient } from '@supabase/supabase-js'; import type { SupportedAiProvider, AiSpecEnricherConfig } from './ai-spec-enricher.js'; +import type { ScrapedReviewer } from './types.js'; /** * 5-Category Classification for Reviews across Products & Services @@ -39,6 +40,8 @@ export interface ReviewInputItem { downvotes?: number; report_count?: number; platform_created_at?: string | null; + /** Reviewer metadata from the scrape, when the platform exposes it. */ + reviewer?: ScrapedReviewer | null; } export interface AnalyzedReviewResult { @@ -179,6 +182,39 @@ export function computeReviewWeight(params: { return Number(Math.max(0, Math.min(3.0, weight)).toFixed(2)); } +const DAY_MS = 24 * 60 * 60 * 1000; + +/** Age of the reviewer's account when the review was written, in days. */ +function accountAgeDays(r: ReviewInputItem): number | null { + const joined = Date.parse(r.reviewer?.joinedAt || ''); + if (!Number.isFinite(joined)) return null; + // Age at the time of writing; without a review date we would be measuring today's age. + const written = Date.parse(r.platform_created_at || ''); + if (!Number.isFinite(written)) return null; + const days = (written - joined) / DAY_MS; + return days >= 0 ? days : null; +} + +/** `unknown` when the platform does not expose the join date. */ +export function accountAgeBucket(r: ReviewInputItem): ReviewSignals['account_age_bucket'] { + const days = accountAgeDays(r); + if (days === null) return 'unknown'; + if (days < 30) return 'low'; + if (days < 365) return 'medium'; + return 'high'; +} + +/** + * 0..1, higher = more established reviewer. Log-scaled on the account's total review + * count (0 -> 0, 9 -> 0.5, 99+ -> 1). Falls back to the old neutral prior when unknown. + */ +export function reviewerHistoryScore(r: ReviewInputItem): number { + if (r.reviewer_type === 'expert') return 1.0; + const total = r.reviewer?.totalReviews; + if (typeof total !== 'number' || !Number.isFinite(total) || total < 0) return 0.7; + return clamp01(Math.log10(1 + total) / 2); +} + function normalizeTextForDedupe(text: string): string { return text .toLowerCase() @@ -369,8 +405,8 @@ export class AiReviewAnalyzer { signals: { burst_pattern_score: clamp01(burstScore), duplicate_text_ratio: clamp01(duplicateRatio), - account_age_bucket: r.is_verified_buyer === false ? 'low' : 'medium', - reviewer_history_score: r.reviewer_type === 'expert' ? 1.0 : 0.7, + account_age_bucket: accountAgeBucket(r), + reviewer_history_score: reviewerHistoryScore(r), language_marker_score: clamp01(languageMarker), }, }; @@ -446,6 +482,8 @@ Classify each review into one of 5 exact flag_category values: 4. "competitor_sabotage": Malicious 1-2 star attack promoting a rival shop or baseless defamation. Ignored in rating, protects seller. 5. "review_bombing": Review left after cancelling an order without using the product/service, or mob brigading. Ignored in rating. +Reviewer fields (verified_purchase, reviewer_anonymous, reviewer_account_age_days, reviewer_total_reviews, repeat_purchase) are null when the platform does not expose them; null is NOT evidence of fraud. A brand-new account with no other reviews posting generic praise is supporting evidence, never proof on its own. + Return ONLY valid JSON matching: { "reviews": [ @@ -480,7 +518,14 @@ Return ONLY valid JSON matching: author: r.author_name, rating_star: r.rating_star, comment: r.comment, - is_verified_buyer: r.is_verified_buyer ?? true, + verified_purchase: r.reviewer?.verifiedPurchase ?? r.is_verified_buyer ?? null, + reviewer_anonymous: r.reviewer?.isAnonymous ?? null, + reviewer_account_age_days: (() => { + const days = accountAgeDays(r); + return days === null ? null : Math.round(days); + })(), + reviewer_total_reviews: r.reviewer?.totalReviews ?? null, + repeat_purchase: r.reviewer?.repeatPurchase ?? null, heuristic_hint: batchBaseline[idx]?.flag_category, })), }); diff --git a/apps/importer/src/shopee/shopee-to-supabase.ts b/apps/importer/src/shopee/shopee-to-supabase.ts index c1b79a1..8496ec6 100644 --- a/apps/importer/src/shopee/shopee-to-supabase.ts +++ b/apps/importer/src/shopee/shopee-to-supabase.ts @@ -6,11 +6,11 @@ import crypto from 'node:crypto'; import dotenv from 'dotenv'; import { createClient, SupabaseClient } from '@supabase/supabase-js'; -import type { ShopeeProduct, SupportedEcommercePlatform } from './types.js'; +import type { ShopeeProduct, ShopeeReview, SupportedEcommercePlatform } from './types.js'; import { ProductNormalizer } from './product-normalizer.js'; import { EntityMatcher } from './entity-matcher.js'; import { AiSpecEnricher, type AiSpecEnricherConfig } from './ai-spec-enricher.js'; -import { AiReviewAnalyzer } from './ai-review-analyzer.js'; +import { AiReviewAnalyzer, type ReviewInputItem } from './ai-review-analyzer.js'; dotenv.config(); @@ -18,6 +18,17 @@ function hashEntity(val: string): string { return '0x' + crypto.createHash('sha256').update(val.toLowerCase()).digest('hex'); } +function platformReviewIdFor(r: ShopeeReview): string { + return ( + r.reviewId || + 'rv-' + + crypto + .createHash('sha256') + .update(`${r.author || ''}|${r.comment || ''}`) + .digest('hex') + ); +} + export class ShopeeIngestionPipeline { private supabase: SupabaseClient; private normalizer: ProductNormalizer; @@ -267,19 +278,15 @@ export class ShopeeIngestionPipeline { if (product.reviews && product.reviews.length > 0) { const reviewRows = product.reviews.map(r => ({ product_id: productId, - platform_review_id: - r.reviewId || - 'rv-' + - crypto - .createHash('sha256') - .update(`${r.author || ''}|${r.comment || ''}`) - .digest('hex'), + platform_review_id: platformReviewIdFor(r), author_name: r.author, author_avatar: r.authorAvatar || null, rating_star: r.ratingStar, comment: r.comment, variation: r.variation || null, images: r.images || [], + // null = the platform does not say (the column default `true` would claim a purchase we cannot confirm) + is_verified_buyer: r.reviewer?.verifiedPurchase ?? null, platform_created_at: r.createdAt, })); @@ -343,6 +350,24 @@ export class ShopeeIngestionPipeline { ); } } + // Rows stored before the platform's purchase flag was captured still carry the + // column default; refresh it for reviews already in the DB. + for (const verified of [true, false]) { + const ids = product.reviews + .filter(r => r.reviewer?.verifiedPurchase === verified) + .map(platformReviewIdFor); + if (ids.length === 0) continue; + const { error: verifiedErr } = await this.supabase + .from('reviews') + .update({ is_verified_buyer: verified }) + .eq('product_id', productId) + .in('platform_review_id', ids); + if (verifiedErr) { + console.warn( + `[MultiPlatformIngest:${platform}] Could not refresh is_verified_buyer: ${verifiedErr.message}` + ); + } + } await report?.('reviews'); } @@ -351,12 +376,27 @@ export class ShopeeIngestionPipeline { const { data: storedReviews, error: storedErr } = await this.supabase .from('reviews') .select( - 'id, author_name, rating_star, comment, variation, images, is_verified_buyer, reviewer_type, upvotes, downvotes, report_count, platform_created_at' + 'id, platform_review_id, author_name, rating_star, comment, variation, images, is_verified_buyer, reviewer_type, upvotes, downvotes, report_count, platform_created_at' ) .eq('product_id', productId) .order('created_at', { ascending: false }) .limit(200); + // Reviewer metadata is not persisted yet; attach what this scrape saw so the analyzer can use it. + const reviewerByPlatformId = new Map( + (product.reviews || []) + .filter(r => r.reviewer) + .map(r => [platformReviewIdFor(r), r.reviewer] as const) + ); + const analysisInput: ReviewInputItem[] = (storedReviews || []).map( + (row: ReviewInputItem & { platform_review_id?: string | null }) => ({ + ...row, + reviewer: row.platform_review_id + ? reviewerByPlatformId.get(row.platform_review_id) + : undefined, + }) + ); + if (storedErr) { console.warn( `[MultiPlatformIngest:${platform}] Review analysis skipped (fetch failed): ${storedErr.message}` @@ -374,7 +414,7 @@ export class ShopeeIngestionPipeline { productId, masterId, merchantId, - reviews: storedReviews, + reviews: analysisInput, entityName: candidate.name, entityType: 'product', platform, diff --git a/apps/importer/src/shopee/test-review-analyzer.ts b/apps/importer/src/shopee/test-review-analyzer.ts index 594deff..e5e00cf 100644 --- a/apps/importer/src/shopee/test-review-analyzer.ts +++ b/apps/importer/src/shopee/test-review-analyzer.ts @@ -6,7 +6,9 @@ import http from 'node:http'; import assert from 'node:assert/strict'; import { AiReviewAnalyzer, + accountAgeBucket, computeReviewWeight, + reviewerHistoryScore, type AnalyzedReviewResult, type ReviewFlagCategory, type ReviewInputItem, @@ -64,7 +66,12 @@ async function run() { 0, 'abuse reports must drive weight to 0' ); - for (const cat of ['low_value_coin_spam', 'seeding_fake', 'competitor_sabotage', 'review_bombing'] as const) { + for (const cat of [ + 'low_value_coin_spam', + 'seeding_fake', + 'competitor_sabotage', + 'review_bombing', + ] as const) { assert.equal( computeReviewWeight({ upvotes: 50, flagCategory: cat, verdict: 'authentic' }), 0, @@ -79,17 +86,49 @@ async function run() { const reviews: ReviewInputItem[] = [ { id: 'coin', author_name: 'a1', rating_star: 5, comment: 'Nhận xu nhé shop ơi' }, - { id: 'seed1', author_name: 'u_seed_1', rating_star: 5, comment: 'Shop uy tín giao hàng nhanh đóng gói cẩn thận' }, - { id: 'seed2', author_name: 'u_seed_2', rating_star: 5, comment: 'Shop uy tín giao hàng nhanh đóng gói cẩn thận' }, - { id: 'sabo', author_name: 'rival', rating_star: 1, comment: 'Hàng tệ, ib mình bán rẻ hơn nhé' }, - { id: 'bomb', author_name: 'b1', rating_star: 1, comment: 'Mới đặt đã hủy đơn rồi, vào đánh 1 sao cho bõ ghét' }, - { id: 'good', author_name: 'g1', rating_star: 5, comment: 'Máy chạy mượt, pin trâu dùng cả ngày, màn hình rất đẹp', upvotes: 10 }, - { id: 'down', author_name: 'g2', rating_star: 4, comment: 'Camera tạm ổn nhưng loa hơi nhỏ so với tầm giá', downvotes: 4 }, + { + id: 'seed1', + author_name: 'u_seed_1', + rating_star: 5, + comment: 'Shop uy tín giao hàng nhanh đóng gói cẩn thận', + }, + { + id: 'seed2', + author_name: 'u_seed_2', + rating_star: 5, + comment: 'Shop uy tín giao hàng nhanh đóng gói cẩn thận', + }, + { + id: 'sabo', + author_name: 'rival', + rating_star: 1, + comment: 'Hàng tệ, ib mình bán rẻ hơn nhé', + }, + { + id: 'bomb', + author_name: 'b1', + rating_star: 1, + comment: 'Mới đặt đã hủy đơn rồi, vào đánh 1 sao cho bõ ghét', + }, + { + id: 'good', + author_name: 'g1', + rating_star: 5, + comment: 'Máy chạy mượt, pin trâu dùng cả ngày, màn hình rất đẹp', + upvotes: 10, + }, + { + id: 'down', + author_name: 'g2', + rating_star: 4, + comment: 'Camera tạm ổn nhưng loa hơi nhỏ so với tầm giá', + downvotes: 4, + }, ]; const analyzer = heuristicOnly(); const res = analyzer.analyzeWithHeuristics(reviews); - const byId = new Map(res.map((r) => [r.review_id, r])); + const byId = new Map(res.map(r => [r.review_id, r])); assert.equal(byId.get('coin')!.flag_category, 'low_value_coin_spam'); assert.equal(byId.get('coin')!.include_in_rating, false); @@ -100,24 +139,98 @@ async function run() { assert.equal(byId.get('good')!.flag_category, 'authentic'); assert.ok(byId.get('good')!.helpfulness_weight > 1.5); assert.equal(byId.get('down')!.flag_category, 'authentic'); - assert.equal(byId.get('down')!.include_in_rating, false, 'heavily down-voted authentic review is zeroed'); + assert.equal( + byId.get('down')!.include_in_rating, + false, + 'heavily down-voted authentic review is zeroed' + ); for (const r of res) { - assert.ok(signalsValid(r.signals as unknown as Record), `signals for ${r.review_id} must satisfy rate.signals_valid`); + assert.ok( + signalsValid(r.signals as unknown as Record), + `signals for ${r.review_id} must satisfy rate.signals_valid` + ); assert.ok(r.fake_probability >= 0 && r.fake_probability <= 1); } - console.log('✅ [Heuristics] coin_spam / seeding / sabotage / bombing / authentic + vote weights OK'); + console.log( + '✅ [Heuristics] coin_spam / seeding / sabotage / bombing / authentic + vote weights OK' + ); console.log('✅ [Heuristics] every signals payload satisfies the DB CHECK rate.signals_valid'); + // Reviewer-derived signals come from scraped reviewer metadata, not constants. + const base: ReviewInputItem = { + id: 'rv', + rating_star: 5, + comment: 'ok', + platform_created_at: '2026-10-01T00:00:00.000Z', + }; + assert.equal(accountAgeBucket(base), 'unknown', 'no join date => unknown, not a guess'); + assert.equal( + accountAgeBucket({ ...base, reviewer: { joinedAt: '2026-09-25T00:00:00.000Z' } }), + 'low' + ); + assert.equal( + accountAgeBucket({ ...base, reviewer: { joinedAt: '2026-03-01T00:00:00.000Z' } }), + 'medium' + ); + assert.equal( + accountAgeBucket({ ...base, reviewer: { joinedAt: '2021-01-01T00:00:00.000Z' } }), + 'high' + ); + assert.equal( + accountAgeBucket({ ...base, reviewer: { joinedAt: '2027-01-01T00:00:00.000Z' } }), + 'unknown', + 'join date after the review is bad data, not a new account' + ); + assert.equal( + accountAgeBucket({ + id: 'rv', + rating_star: 5, + reviewer: { joinedAt: '2021-01-01T00:00:00.000Z' }, + }), + 'unknown', + 'no review date => cannot tell the account age at writing time' + ); + assert.equal(reviewerHistoryScore(base), 0.7, 'unknown history keeps the neutral prior'); + assert.equal(reviewerHistoryScore({ ...base, reviewer: { totalReviews: 0 } }), 0); + assert.equal(reviewerHistoryScore({ ...base, reviewer: { totalReviews: 9 } }), 0.5); + assert.equal(reviewerHistoryScore({ ...base, reviewer: { totalReviews: 5000 } }), 1); + assert.equal(reviewerHistoryScore({ ...base, reviewer_type: 'expert' }), 1); + const [withReviewer] = heuristicOnly().analyzeWithHeuristics([ + { ...base, reviewer: { joinedAt: '2026-09-28T00:00:00.000Z', totalReviews: 1 } }, + ]); + assert.equal(withReviewer!.signals.account_age_bucket, 'low'); + assert.equal(withReviewer!.signals.reviewer_history_score, 0.151); + assert.ok(signalsValid(withReviewer!.signals as unknown as Record)); + console.log( + '✅ [Reviewer] account age bucket & history score derived from scraped reviewer metadata' + ); + console.log('\n================================================================'); console.log('3. JReviews dual rating (Expert vs weighted authentic User)'); console.log('================================================================'); const jr: ReviewInputItem[] = [ - { id: 'u1', author_name: 'x1', rating_star: 5, comment: 'Dùng được hai tuần rất hài lòng về hiệu năng' }, - { id: 'u2', author_name: 'x2', rating_star: 5, comment: 'Pin bền hơn máy cũ của mình rất nhiều thật sự' }, + { + id: 'u1', + author_name: 'x1', + rating_star: 5, + comment: 'Dùng được hai tuần rất hài lòng về hiệu năng', + }, + { + id: 'u2', + author_name: 'x2', + rating_star: 5, + comment: 'Pin bền hơn máy cũ của mình rất nhiều thật sự', + }, { id: 'spam', author_name: 'x3', rating_star: 1, comment: 'nhận xu' }, - { id: 'exp', author_name: 'Tech Reviewer', rating_star: 4, comment: 'Đánh giá chuyên sâu sau một tháng sử dụng máy', reviewer_type: 'expert' }, + { + id: 'exp', + author_name: 'Tech Reviewer', + rating_star: 4, + comment: 'Đánh giá chuyên sâu sau một tháng sử dụng máy', + reviewer_type: 'expert', + }, ]; const jrRes = heuristicOnly().analyzeWithHeuristics(jr); const summary = heuristicOnly().computeJReviewsSummary(jr, jrRes); @@ -125,11 +238,17 @@ async function run() { assert.equal(summary.total_reviews_analyzed, 4); assert.equal(summary.authentic_review_count, 3); assert.equal(summary.ignored_review_count, 1); - assert.equal(summary.user_rating_weighted, 5, 'ignored spam must not drag the weighted user rating'); + assert.equal( + summary.user_rating_weighted, + 5, + 'ignored spam must not drag the weighted user rating' + ); assert.equal(summary.expert_rating, 4); assert.equal(summary.overall_rating, 4.6, 'overall = 40% expert + 60% weighted user'); assert.equal(summary.breakdown_by_category.low_value_coin_spam, 1); - console.log(`✅ [JReviews] user=${summary.user_rating_weighted} expert=${summary.expert_rating} overall=${summary.overall_rating}`); + console.log( + `✅ [JReviews] user=${summary.user_rating_weighted} expert=${summary.expert_rating} overall=${summary.overall_rating}` + ); console.log('\n================================================================'); console.log('4. Merchant risk: fair to sellers, harsh on seeding & dump pricing'); @@ -143,7 +262,13 @@ async function run() { include_in_rating: flag === 'authentic', helpfulness_weight: flag === 'authentic' ? 1 : 0, reason: '', - signals: { burst_pattern_score: 0, duplicate_text_ratio: 0, account_age_bucket: 'medium', reviewer_history_score: 0.5, language_marker_score: 0 }, + signals: { + burst_pattern_score: 0, + duplicate_text_ratio: 0, + account_age_bucket: 'medium', + reviewer_history_score: 0.5, + language_marker_score: 0, + }, }); const clean = analyzer.evaluateMerchantRisk({ @@ -165,10 +290,19 @@ async function run() { const sabotaged = analyzer.evaluateMerchantRisk({ platform: 'shopee', isOfficial: false, - reviewResults: [mk('competitor_sabotage'), mk('review_bombing'), mk('low_value_coin_spam'), mk('authentic')], + reviewResults: [ + mk('competitor_sabotage'), + mk('review_bombing'), + mk('low_value_coin_spam'), + mk('authentic'), + ], }); - assert.equal(sabotaged.risk_score, 15, 'sabotage / bombing / coin spam must NOT penalise the seller'); - assert.ok(sabotaged.reasons.some((r) => r.includes('bảo vệ'))); + assert.equal( + sabotaged.risk_score, + 15, + 'sabotage / bombing / coin spam must NOT penalise the seller' + ); + assert.ok(sabotaged.reasons.some(r => r.includes('bảo vệ'))); const dump = analyzer.evaluateMerchantRisk({ platform: 'shopee', @@ -187,8 +321,14 @@ async function run() { canonicalMinPrice: 28_500_000, reviewResults: [mk('authentic')], }); - assert.equal(mall.price_anomaly_detected, false, 'official stores are exempt from price-anomaly scoring'); - console.log(`✅ [Risk] clean=${clean.risk_score} seeded=${seeded.risk_score} sabotaged=${sabotaged.risk_score} dump=${dump.risk_score} mall=${mall.risk_score}`); + assert.equal( + mall.price_anomaly_detected, + false, + 'official stores are exempt from price-anomaly scoring' + ); + console.log( + `✅ [Risk] clean=${clean.risk_score} seeded=${seeded.risk_score} sabotaged=${sabotaged.risk_score} dump=${dump.risk_score} mall=${mall.risk_score}` + ); console.log('\n================================================================'); console.log('5. Tier 2: LLM batch refinement via mock OpenAI-compatible server'); @@ -198,7 +338,7 @@ async function run() { const server = http.createServer((req, res) => { if (req.method === 'POST' && req.url === '/v1/chat/completions') { let body = ''; - req.on('data', (c) => (body += c)); + req.on('data', c => (body += c)); req.on('end', () => { llmCalls++; const payload = JSON.parse(body); @@ -209,7 +349,8 @@ async function run() { flag_category: r.review_id === 'llm-seed' ? 'seeding_fake' : 'authentic', verdict: r.review_id === 'llm-seed' ? 'fake' : 'authentic', fake_probability: r.review_id === 'llm-seed' ? 0.93 : 0.04, - reason: r.review_id === 'llm-seed' ? 'Văn phong quảng cáo, kịch bản lặp' : 'Trải nghiệm thật', + reason: + r.review_id === 'llm-seed' ? 'Văn phong quảng cáo, kịch bản lặp' : 'Trải nghiệm thật', })), pros: ['Pin trâu'], cons: ['Loa nhỏ'], @@ -223,7 +364,7 @@ async function run() { res.writeHead(404); res.end(); }); - await new Promise((r) => server.listen(0, '127.0.0.1', () => r())); + await new Promise(r => server.listen(0, '127.0.0.1', () => r())); const port = (server.address() as { port: number }).port; try { @@ -235,18 +376,36 @@ async function run() { }); const input: ReviewInputItem[] = [ - { id: 'llm-seed', author_name: 'n1', rating_star: 5, comment: 'Sản phẩm xuất sắc hoàn hảo mọi mặt nên mua ngay hôm nay' }, - { id: 'llm-real', author_name: 'n2', rating_star: 4, comment: 'Pin dùng được cả ngày nhưng loa hơi nhỏ thôi' }, + { + id: 'llm-seed', + author_name: 'n1', + rating_star: 5, + comment: 'Sản phẩm xuất sắc hoàn hảo mọi mặt nên mua ngay hôm nay', + }, + { + id: 'llm-real', + author_name: 'n2', + rating_star: 4, + comment: 'Pin dùng được cả ngày nhưng loa hơi nhỏ thôi', + }, { id: 'llm-coin', author_name: 'n3', rating_star: 5, comment: 'nhận xu' }, ]; const { results, summary: s } = await llm.analyzeReviews(input, { entityName: 'Test Phone' }); - const m = new Map(results.map((r) => [r.review_id, r])); + const m = new Map(results.map(r => [r.review_id, r])); assert.equal(llmCalls, 1, 'one batch => one LLM call'); - assert.equal(m.get('llm-seed')!.flag_category, 'seeding_fake', 'LLM may upgrade an authentic-by-heuristic review'); + assert.equal( + m.get('llm-seed')!.flag_category, + 'seeding_fake', + 'LLM may upgrade an authentic-by-heuristic review' + ); assert.equal(m.get('llm-seed')!.include_in_rating, false); assert.equal(m.get('llm-real')!.flag_category, 'authentic'); - assert.equal(m.get('llm-coin')!.flag_category, 'low_value_coin_spam', 'high-confidence heuristic must not be overridden by LLM'); + assert.equal( + m.get('llm-coin')!.flag_category, + 'low_value_coin_spam', + 'high-confidence heuristic must not be overridden by LLM' + ); assert.deepEqual(s.pros, ['Pin trâu']); assert.deepEqual(s.cons, ['Loa nhỏ']); assert.equal(s.authentic_review_count, 1); @@ -256,7 +415,8 @@ async function run() { console.log('6. persistAnalysisToSupabase payload shape (stubbed client)'); console.log('================================================================'); - const calls: Array<{ table: string; op: string; payload: any; col?: string; val?: string }> = []; + const calls: Array<{ table: string; op: string; payload: any; col?: string; val?: string }> = + []; const stub = { from: (table: string) => ({ upsert: async (payload: any) => { @@ -283,25 +443,57 @@ async function run() { isOfficial: false, }); - const flagUpsert = calls.find((c) => c.table === 'review_flags' && c.op === 'upsert')!; + const flagUpsert = calls.find(c => c.table === 'review_flags' && c.op === 'upsert')!; assert.equal(flagUpsert.payload.length, 3); for (const row of flagUpsert.payload) { - assert.ok(['authentic', 'low_value_coin_spam', 'seeding_fake', 'competitor_sabotage', 'review_bombing'].includes(row.flag_category)); + assert.ok( + [ + 'authentic', + 'low_value_coin_spam', + 'seeding_fake', + 'competitor_sabotage', + 'review_bombing', + ].includes(row.flag_category) + ); assert.ok(['authentic', 'suspicious', 'fake', 'unanalyzed'].includes(row.verdict)); assert.ok(signalsValid(row.signals)); assert.match(row.input_hash, /^0x[0-9a-f]{64}$/); } - assert.ok(calls.some((c) => c.table === 'review_analysis' && c.op === 'upsert' && c.payload.product_id === 'prod-1')); - assert.ok(calls.some((c) => c.table === 'master_products' && c.op === 'update' && c.val === 'master-1' && 'user_rating_weighted' in c.payload)); - const merchUpdate = calls.find((c) => c.table === 'merchants' && c.op === 'update' && c.val === 'merch-1'); + assert.ok( + calls.some( + c => c.table === 'review_analysis' && c.op === 'upsert' && c.payload.product_id === 'prod-1' + ) + ); + assert.ok( + calls.some( + c => + c.table === 'master_products' && + c.op === 'update' && + c.val === 'master-1' && + 'user_rating_weighted' in c.payload + ) + ); + const merchUpdate = calls.find( + c => c.table === 'merchants' && c.op === 'update' && c.val === 'merch-1' + ); assert.ok(merchUpdate, 'merchants update called'); assert.equal(typeof merchUpdate.payload.review_risk_score, 'number'); assert.equal(typeof merchUpdate.payload.review_risk_breakdown, 'object'); assert.equal(typeof merchUpdate.payload.review_risk_updated_at, 'string'); - assert.equal('risk_score' in merchUpdate.payload, false, 'must NOT overwrite DOSafe risk_score'); - assert.equal(calls.filter((c) => c.table === 'reviews' && c.op === 'update').length, 3, 'helpfulness_weight written per review'); + assert.equal( + 'risk_score' in merchUpdate.payload, + false, + 'must NOT overwrite DOSafe risk_score' + ); + assert.equal( + calls.filter(c => c.table === 'reviews' && c.op === 'update').length, + 3, + 'helpfulness_weight written per review' + ); assert.ok(out.merchantRisk); - console.log('✅ [Persist] review_flags / review_analysis / master_products / merchants (review_risk_score isolated) / reviews payloads valid'); + console.log( + '✅ [Persist] review_flags / review_analysis / master_products / merchants (review_risk_score isolated) / reviews payloads valid' + ); console.log('\n================================================================'); console.log('7. castReviewVote (rate.review_votes per-user vote & report)'); @@ -364,7 +556,7 @@ async function run() { console.log('\n🎉 ALL AI REVIEW ANALYZER TESTS PASSED!'); } -run().catch((err) => { +run().catch(err => { console.error('❌ Test failed:', err); process.exit(1); }); diff --git a/apps/importer/src/shopee/types.ts b/apps/importer/src/shopee/types.ts index d6b0db2..dfc69a9 100644 --- a/apps/importer/src/shopee/types.ts +++ b/apps/importer/src/shopee/types.ts @@ -16,6 +16,33 @@ export type SupportedEcommercePlatform = | 'didongviet' | 'phongvu'; +/** + * Reviewer metadata scraped alongside a review. What each platform exposes differs + * (Shopee: user id + shop page; Tiki: join date, review count, purchase flag; + * retail chains: only a "đã mua" badge), so every field is optional and + * `undefined` means "the platform does not say", never "no". + */ +export interface ScrapedReviewer { + /** Pseudonymous stable key (keyed hash of platform + platform user id). Never the raw id. */ + key?: string; + /** Platform marks the review as anonymous / hides the reviewer's identity. */ + isAnonymous?: boolean; + /** Public profile page on the platform, when one exists. */ + profileUrl?: string; + /** Account creation time on the platform (ISO 8601). */ + joinedAt?: string; + /** Total reviews this account has written on the platform. */ + totalReviews?: number; + /** Helpful / thank votes this account has received across its reviews. */ + helpfulVotes?: number; + /** Platform confirms this reviewer bought the item. */ + verifiedPurchase?: boolean; + /** Purchase time as reported by the platform (ISO 8601). */ + purchasedAt?: string; + /** Platform flags the reviewer as a repeat buyer of this item/shop. */ + repeatPurchase?: boolean; +} + export interface ShopeeReview { reviewId: string; author: string; @@ -27,6 +54,9 @@ export interface ShopeeReview { images: string[]; videos?: string[]; sellerReply?: string; + /** Helpful/like count shown on the platform for this review. */ + likeCount?: number; + reviewer?: ScrapedReviewer; } export interface ShopeeProduct { diff --git a/apps/web/src/lib/rate-api.ts b/apps/web/src/lib/rate-api.ts index a3b1959..afc6426 100644 --- a/apps/web/src/lib/rate-api.ts +++ b/apps/web/src/lib/rate-api.ts @@ -28,6 +28,37 @@ export function getRateClient(): RateSchemaClient { return _client; } +/** + * PostgREST caps every response at the project's max-rows (1000 by default) and + * silently drops the rest, so unbounded table reads must be paged. Pages advance by + * the number of rows actually returned, which also works if max-rows is lowered. + * On error, logs and returns what was read so far (matching the old ignore-error reads). + */ +async function fetchAllRows( + label: string, + buildPage: ( + from: number, + to: number + ) => PromiseLike<{ data: T[] | null; error: { message: string } | null }>, + pageSize = 1000 +): Promise { + const rows: T[] = []; + for (let from = 0; ;) { + const { data, error } = await buildPage(from, from + pageSize - 1); + if (error) { + console.error( + `rate-api: paged read of ${label} failed at row ${from}:`, + error.message + ); + return rows; + } + const page = data ?? []; + if (page.length === 0) return rows; + rows.push(...page); + from += page.length; + } +} + export type ReviewFlagCategory = | 'authentic' | 'low_value_coin_spam' @@ -1600,16 +1631,27 @@ export async function getMerchantsList(params?: { mQuery = mQuery.limit(params.limit); } - const [{ data: merchantRows }, { data: prodRows }, { data: reviewRows }] = - await Promise.all([ - mQuery, - supabase.from('products').select('id, merchant_id, price_current'), + const [{ data: merchantRows }, prodRows, reviewRows] = await Promise.all([ + mQuery, + fetchAllRows('products', (from, to) => + supabase + .from('products') + .select('id, merchant_id, price_current') + .order('id') + .range(from, to) + ), + // Only the flag fields feed computeDynamicReviewRisk; keep this read narrow + // because it pages through every review. + fetchAllRows('reviews', (from, to) => supabase .from('reviews') - .select('*, review_flags(*)') - .order('created_at', { ascending: false }) - .limit(1000), - ]); + .select( + 'id, product_id, review_flags(verdict, flag_category, include_in_rating)' + ) + .order('id') + .range(from, to) + ), + ]); const merchants = (merchantRows as unknown as RawMerchantSafeRow[]) || []; if (merchants.length === 0) return []; @@ -2074,11 +2116,21 @@ export async function getRatePlatformStats(): Promise { .from('categories') .select('id, parent_id, name, slug, icon, display_order') .order('display_order', { ascending: true }), - supabase - .from('master_products') - .select('id, slug, name, category_id') - .eq('status', 'published'), - supabase.from('products').select('id, title, cluster_id'), + fetchAllRows('master_products', (from, to) => + supabase + .from('master_products') + .select('id, slug, name, category_id') + .eq('status', 'published') + .order('id') + .range(from, to) + ).then((data) => ({ data })), + fetchAllRows('products', (from, to) => + supabase + .from('products') + .select('id, title, cluster_id') + .order('id') + .range(from, to) + ).then((data) => ({ data })), supabase.from('merchants').select('id', { count: 'exact', head: true }), supabase.from('price_history').select('id', { count: 'exact', head: true }), supabase