Kingsmai/pokopiawiki.tootaio.com
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(moderation): add AI moderation for user-generated content

Browse Source 
Add AI moderation settings, caching, and status tracking
Require AI approval for Life Posts, Comments, and Discussions
Implement language filtering and moderation status UI
Add retry mechanism for failed moderation checks
This commit is contained in:
xiaomai
2026-05-03 17:08:51 +08:00
parent 590bd6a0ae
commit 18baf7b513
12 changed files with 2217 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

125
backend/db/schema.sql
View File

@@ -134,6 +134,49 @@ CREATE TABLE IF NOT EXISTS user_roles (
CREATE INDEX IF NOT EXISTS user_roles_role_id_idx
ON user_roles(role_id, user_id);
CREATE TABLE IF NOT EXISTS ai_moderation_settings (
id boolean PRIMARY KEY DEFAULT true CHECK (id = true),
enabled boolean NOT NULL DEFAULT true,
api_format text NOT NULL DEFAULT 'gemini-generate-content' CHECK (api_format IN ('gemini-generate-content', 'openai-chat-completions')),
auth_mode text NOT NULL DEFAULT 'bearer-token' CHECK (auth_mode IN ('query-key', 'bearer-token')),
endpoint text NOT NULL DEFAULT 'https://ai.example.com/v1beta',
api_key text NOT NULL DEFAULT '',
model text NOT NULL DEFAULT 'gemini-2.0-flash-lite',
requests_per_minute integer NOT NULL DEFAULT 10 CHECK (requests_per_minute BETWEEN 1 AND 60),
updated_by_user_id integer REFERENCES users(id) ON DELETE SET NULL,
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now(),
CHECK (length(endpoint) BETWEEN 1 AND 300),
CHECK (length(model) BETWEEN 1 AND 120)
);
ALTER TABLE ai_moderation_settings
ADD COLUMN IF NOT EXISTS api_format text NOT NULL DEFAULT 'gemini-generate-content' CHECK (api_format IN ('gemini-generate-content', 'openai-chat-completions')),
ADD COLUMN IF NOT EXISTS auth_mode text NOT NULL DEFAULT 'bearer-token' CHECK (auth_mode IN ('query-key', 'bearer-token'));
INSERT INTO ai_moderation_settings (id)
VALUES (true)
ON CONFLICT (id) DO NOTHING;
UPDATE ai_moderation_settings
SET api_format = 'gemini-generate-content',
auth_mode = 'bearer-token',
updated_at = now()
WHERE api_format = 'openai-chat-completions'
AND auth_mode = 'query-key'
AND endpoint ~* '/v1beta/?$';
CREATE TABLE IF NOT EXISTS ai_moderation_cache (
content_hash text NOT NULL,
model text NOT NULL,
status text NOT NULL CHECK (status IN ('approved', 'rejected')),
language_code text REFERENCES languages(code) ON DELETE SET NULL,
checked_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (content_hash, model),
CHECK (length(content_hash) BETWEEN 32 AND 128),
CHECK (length(model) BETWEEN 1 AND 120)
);
INSERT INTO permissions (key, name, description, category, system_permission)
VALUES
('admin.access', 'Access admin', 'Open the management area.', 'Admin', true),
@@ -155,6 +198,8 @@ VALUES
('admin.languages.order', 'Order languages', 'Reorder languages.', 'Languages', true),
('admin.wordings.read', 'View system wordings', 'View system wording values.', 'System wordings', true),
('admin.wordings.update', 'Update system wordings', 'Edit system wording values.', 'System wordings', true),
('admin.ai-moderation.read', 'View AI moderation settings', 'View AI moderation configuration.', 'AI moderation', true),
('admin.ai-moderation.update', 'Update AI moderation settings', 'Edit AI moderation configuration.', 'AI moderation', true),
('admin.config.read', 'View system config', 'View management configuration records.', 'System config', true),
('admin.config.create', 'Create system config', 'Create management configuration records.', 'System config', true),
('admin.config.update', 'Update system config', 'Edit management configuration records.', 'System config', true),
@@ -236,6 +281,8 @@ JOIN permissions p ON p.key = ANY (ARRAY[
'admin.languages.order',
'admin.wordings.read',
'admin.wordings.update',
'admin.ai-moderation.read',
'admin.ai-moderation.update',
'admin.config.read',
'admin.config.create',
'admin.config.update',
@@ -283,7 +330,17 @@ WHERE r.key = 'admin'
SELECT 1
FROM role_permissions existing_role_permission
WHERE existing_role_permission.role_id = r.id
)
)
ON CONFLICT DO NOTHING;
INSERT INTO role_permissions (role_id, permission_id)
SELECT r.id, p.id
FROM roles r
JOIN permissions p ON p.key = ANY (ARRAY[
'admin.ai-moderation.read',
'admin.ai-moderation.update'
])
WHERE r.key = 'admin'
ON CONFLICT DO NOTHING;
INSERT INTO role_permissions (role_id, permission_id)
@@ -476,6 +533,12 @@ CREATE TABLE IF NOT EXISTS life_tags (
CREATE TABLE IF NOT EXISTS life_posts (
id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
body text NOT NULL CHECK (length(body) BETWEEN 1 AND 2000),
ai_moderation_status text NOT NULL DEFAULT 'unreviewed' CHECK (ai_moderation_status IN ('unreviewed', 'reviewing', 'approved', 'rejected', 'failed')),
ai_moderation_language_code text REFERENCES languages(code) ON DELETE SET NULL,
ai_moderation_content_hash text,
ai_moderation_checked_at timestamptz,
ai_moderation_retry_count integer NOT NULL DEFAULT 0 CHECK (ai_moderation_retry_count >= 0),
ai_moderation_updated_at timestamptz NOT NULL DEFAULT now(),
created_by_user_id integer REFERENCES users(id) ON DELETE SET NULL,
updated_by_user_id integer REFERENCES users(id) ON DELETE SET NULL,
deleted_by_user_id integer REFERENCES users(id) ON DELETE SET NULL,
@@ -509,6 +572,12 @@ CREATE TABLE IF NOT EXISTS life_post_comments (
post_id integer NOT NULL REFERENCES life_posts(id) ON DELETE CASCADE,
parent_comment_id integer REFERENCES life_post_comments(id) ON DELETE SET NULL,
body text NOT NULL CHECK (length(body) BETWEEN 1 AND 1000),
ai_moderation_status text NOT NULL DEFAULT 'unreviewed' CHECK (ai_moderation_status IN ('unreviewed', 'reviewing', 'approved', 'rejected', 'failed')),
ai_moderation_language_code text REFERENCES languages(code) ON DELETE SET NULL,
ai_moderation_content_hash text,
ai_moderation_checked_at timestamptz,
ai_moderation_retry_count integer NOT NULL DEFAULT 0 CHECK (ai_moderation_retry_count >= 0),
ai_moderation_updated_at timestamptz NOT NULL DEFAULT now(),
created_by_user_id integer REFERENCES users(id) ON DELETE SET NULL,
deleted_by_user_id integer REFERENCES users(id) ON DELETE SET NULL,
deleted_at timestamptz,
@@ -807,6 +876,12 @@ CREATE TABLE IF NOT EXISTS entity_discussion_comments (
entity_id integer NOT NULL,
parent_comment_id integer REFERENCES entity_discussion_comments(id) ON DELETE CASCADE,
body text NOT NULL CHECK (length(body) BETWEEN 1 AND 1000),
ai_moderation_status text NOT NULL DEFAULT 'unreviewed' CHECK (ai_moderation_status IN ('unreviewed', 'reviewing', 'approved', 'rejected', 'failed')),
ai_moderation_language_code text REFERENCES languages(code) ON DELETE SET NULL,
ai_moderation_content_hash text,
ai_moderation_checked_at timestamptz,
ai_moderation_retry_count integer NOT NULL DEFAULT 0 CHECK (ai_moderation_retry_count >= 0),
ai_moderation_updated_at timestamptz NOT NULL DEFAULT now(),
created_by_user_id integer REFERENCES users(id) ON DELETE SET NULL,
deleted_by_user_id integer REFERENCES users(id) ON DELETE SET NULL,
deleted_at timestamptz,
@@ -822,3 +897,51 @@ CREATE INDEX IF NOT EXISTS entity_discussion_comments_parent_idx
CREATE INDEX IF NOT EXISTS entity_discussion_comments_user_idx
ON entity_discussion_comments(created_by_user_id);
ALTER TABLE life_posts
ADD COLUMN IF NOT EXISTS ai_moderation_status text NOT NULL DEFAULT 'unreviewed' CHECK (ai_moderation_status IN ('unreviewed', 'reviewing', 'approved', 'rejected', 'failed')),
ADD COLUMN IF NOT EXISTS ai_moderation_language_code text REFERENCES languages(code) ON DELETE SET NULL,
ADD COLUMN IF NOT EXISTS ai_moderation_content_hash text,
ADD COLUMN IF NOT EXISTS ai_moderation_checked_at timestamptz,
ADD COLUMN IF NOT EXISTS ai_moderation_retry_count integer NOT NULL DEFAULT 0 CHECK (ai_moderation_retry_count >= 0),
ADD COLUMN IF NOT EXISTS ai_moderation_updated_at timestamptz NOT NULL DEFAULT now();
ALTER TABLE life_post_comments
ADD COLUMN IF NOT EXISTS ai_moderation_status text NOT NULL DEFAULT 'unreviewed' CHECK (ai_moderation_status IN ('unreviewed', 'reviewing', 'approved', 'rejected', 'failed')),
ADD COLUMN IF NOT EXISTS ai_moderation_language_code text REFERENCES languages(code) ON DELETE SET NULL,
ADD COLUMN IF NOT EXISTS ai_moderation_content_hash text,
ADD COLUMN IF NOT EXISTS ai_moderation_checked_at timestamptz,
ADD COLUMN IF NOT EXISTS ai_moderation_retry_count integer NOT NULL DEFAULT 0 CHECK (ai_moderation_retry_count >= 0),
ADD COLUMN IF NOT EXISTS ai_moderation_updated_at timestamptz NOT NULL DEFAULT now();
ALTER TABLE entity_discussion_comments
ADD COLUMN IF NOT EXISTS ai_moderation_status text NOT NULL DEFAULT 'unreviewed' CHECK (ai_moderation_status IN ('unreviewed', 'reviewing', 'approved', 'rejected', 'failed')),
ADD COLUMN IF NOT EXISTS ai_moderation_language_code text REFERENCES languages(code) ON DELETE SET NULL,
ADD COLUMN IF NOT EXISTS ai_moderation_content_hash text,
ADD COLUMN IF NOT EXISTS ai_moderation_checked_at timestamptz,
ADD COLUMN IF NOT EXISTS ai_moderation_retry_count integer NOT NULL DEFAULT 0 CHECK (ai_moderation_retry_count >= 0),
ADD COLUMN IF NOT EXISTS ai_moderation_updated_at timestamptz NOT NULL DEFAULT now();
CREATE INDEX IF NOT EXISTS life_posts_ai_moderation_status_idx
ON life_posts(ai_moderation_status, ai_moderation_updated_at, id)
WHERE deleted_at IS NULL;
CREATE INDEX IF NOT EXISTS life_posts_ai_moderation_language_idx
ON life_posts(ai_moderation_language_code, created_at DESC, id DESC)
WHERE deleted_at IS NULL;
CREATE INDEX IF NOT EXISTS life_post_comments_ai_moderation_status_idx
ON life_post_comments(ai_moderation_status, ai_moderation_updated_at, id)
WHERE deleted_at IS NULL;
CREATE INDEX IF NOT EXISTS life_post_comments_ai_moderation_language_idx
ON life_post_comments(ai_moderation_language_code, created_at, id)
WHERE deleted_at IS NULL;
CREATE INDEX IF NOT EXISTS entity_discussion_comments_ai_moderation_status_idx
ON entity_discussion_comments(ai_moderation_status, ai_moderation_updated_at, id)
WHERE deleted_at IS NULL;
CREATE INDEX IF NOT EXISTS entity_discussion_comments_ai_moderation_language_idx
ON entity_discussion_comments(entity_type, entity_id, ai_moderation_language_code, created_at, id)
WHERE deleted_at IS NULL;