From 715ace7ac69e1520f971ed09c4714dbb891b56f5 Mon Sep 17 00:00:00 2001 From: "DESKTOP-L8OUL9F\\gfl0034" <1003614916@qq.com> Date: Thu, 24 Sep 2026 23:07:06 +0800 Subject: [PATCH] =?UTF-8?q?feat(square):=20=E5=B9=BF=E5=9C=BA=20Bot/RSS=20?= =?UTF-8?q?=E5=8F=91=E5=B8=83=E4=B8=8E=E5=AA=92=E4=BD=93=E6=A0=A1=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 支持广场机器人发帖与媒体类型检查,并补充相关迁移与 i18n。 --- internal/infrastructure/media/bot.go | 74 ++++++ internal/infrastructure/media/image.go | 124 ++++++++++ internal/infrastructure/media/image_test.go | 71 ++++++ internal/infrastructure/media/video.go | 148 ++++++++++++ internal/infrastructure/objectstore/ops.go | 4 +- internal/module/meta/logic/enums_wire.go | 1 + internal/module/moment/logic/media_check.go | 55 +++++ internal/module/moment/logic/publish.go | 3 + internal/module/moment/logic/service.go | 18 +- internal/module/square/entity/post.go | 14 ++ internal/module/square/entity/source.go | 18 ++ internal/module/square/logic/bot_publish.go | 83 +++++++ internal/module/square/logic/media_check.go | 56 +++++ internal/module/square/logic/publish.go | 3 + internal/module/square/logic/service.go | 18 +- internal/module/square/logic/visible.go | 4 + internal/module/square/model/bot.go | 103 +++++++++ internal/module/square/model/memory.go | 11 + internal/module/square/model/memory_write.go | 52 +++++ internal/module/square/model/post.go | 26 ++- internal/module/square/model/query.go | 9 +- internal/module/squarebot/logic/.keep | 0 internal/module/squarebot/logic/doc.go | 3 + internal/module/squarebot/logic/media.go | 104 +++++++++ internal/module/squarebot/logic/rss.go | 228 +++++++++++++++++++ internal/module/squarebot/logic/service.go | 213 +++++++++++++++++ internal/pkg/errs/codes.go | 1 + internal/pkg/i18n/catalog_err.go | 3 + internal/pkg/i18n/catalog_validate_upload.go | 5 + internal/pkg/i18n/keys.go | 1 + internal/pkg/media/check.go | 83 +++++++ internal/pkg/media/check_test.go | 45 ++++ internal/pkg/media/limits.go | 77 +++++++ internal/pkg/media/publish.go | 34 +++ migrations/0026_square_bot_rss.sql | 19 ++ 35 files changed, 1702 insertions(+), 9 deletions(-) create mode 100644 internal/infrastructure/media/bot.go create mode 100644 internal/infrastructure/media/image.go create mode 100644 internal/infrastructure/media/image_test.go create mode 100644 internal/infrastructure/media/video.go create mode 100644 internal/module/moment/logic/media_check.go create mode 100644 internal/module/square/entity/source.go create mode 100644 internal/module/square/logic/bot_publish.go create mode 100644 internal/module/square/logic/media_check.go create mode 100644 internal/module/square/model/bot.go create mode 100644 internal/module/squarebot/logic/.keep create mode 100644 internal/module/squarebot/logic/doc.go create mode 100644 internal/module/squarebot/logic/media.go create mode 100644 internal/module/squarebot/logic/rss.go create mode 100644 internal/module/squarebot/logic/service.go create mode 100644 internal/pkg/media/check.go create mode 100644 internal/pkg/media/check_test.go create mode 100644 internal/pkg/media/limits.go create mode 100644 internal/pkg/media/publish.go create mode 100644 migrations/0026_square_bot_rss.sql diff --git a/internal/infrastructure/media/bot.go b/internal/infrastructure/media/bot.go new file mode 100644 index 0000000..70ee20b --- /dev/null +++ b/internal/infrastructure/media/bot.go @@ -0,0 +1,74 @@ +package media + +import ( + "context" + "fmt" + "strings" + + pkgmedia "spark-server/internal/pkg/media" +) + +// BotAsset is compressed media ready for objectstore.Put / PutLarge. +type BotAsset struct { + Data []byte + Width int + Height int + MIME string + Size int64 +} + +// PrepareBotImage compresses RSS/downloaded image bytes to ≤1080p before object storage write. +// Call this after HTTP download and before PutLarge. +func PrepareBotImage(src []byte, opt Options) (*BotAsset, error) { + res, err := CompressImage(src, opt) + if err != nil { + return nil, err + } + return &BotAsset{ + Data: res.Data, Width: res.Width, Height: res.Height, + MIME: res.MIME, Size: int64(len(res.Data)), + }, nil +} + +// PrepareBotAvatar compresses bot avatar to ≤200×200. +func PrepareBotAvatar(src []byte, opt Options) (*BotAsset, error) { + res, err := CompressAvatar(src, opt) + if err != nil { + return nil, err + } + return &BotAsset{ + Data: res.Data, Width: res.Width, Height: res.Height, + MIME: res.MIME, Size: int64(len(res.Data)), + }, nil +} + +// PrepareBotVideo compresses downloaded video to ≤720p (requires ffmpeg). +func PrepareBotVideo(ctx context.Context, src []byte, opt Options) (*BotAsset, error) { + res, err := CompressVideoBytes(ctx, src, opt) + if err != nil { + return nil, err + } + return &BotAsset{ + Data: res.Data, Width: res.Width, Height: res.Height, + MIME: res.MIME, Size: int64(len(res.Data)), + }, nil +} + +// OptionsFromLimits builds compressor options from shared policy limits. +func OptionsFromLimits(lim pkgmedia.Limits, jpegQuality, videoCRF int, ffmpegPath string) Options { + return Options{ + Limits: lim.Normalize(), + JPEGQuality: jpegQuality, + VideoCRF: videoCRF, + FFmpegPath: strings.TrimSpace(ffmpegPath), + } +} + +// MustCompressBotImage is PrepareBotImage with a clearer error prefix for callers. +func MustCompressBotImage(src []byte, opt Options) (*BotAsset, error) { + a, err := PrepareBotImage(src, opt) + if err != nil { + return nil, fmt.Errorf("bot image compress: %w", err) + } + return a, nil +} diff --git a/internal/infrastructure/media/image.go b/internal/infrastructure/media/image.go new file mode 100644 index 0000000..211a215 --- /dev/null +++ b/internal/infrastructure/media/image.go @@ -0,0 +1,124 @@ +// Package media compresses images (stdlib + x/image) and videos (ffmpeg) for bot pipelines. +// Deploy note: video compression requires the `ffmpeg` binary on PATH (or Options.FFmpegPath). +package media + +import ( + "bytes" + "fmt" + "image" + _ "image/gif" + "image/jpeg" + _ "image/png" + "strings" + + "golang.org/x/image/draw" + + pkgmedia "spark-server/internal/pkg/media" +) + +// Options configures compressors. +type Options struct { + Limits pkgmedia.Limits + JPEGQuality int + VideoCRF int + FFmpegPath string // default "ffmpeg" +} + +func (o Options) normalize() Options { + o.Limits = o.Limits.Normalize() + if o.JPEGQuality <= 0 || o.JPEGQuality > 100 { + o.JPEGQuality = pkgmedia.DefaultJPEGQuality + } + if o.VideoCRF <= 0 { + o.VideoCRF = pkgmedia.DefaultVideoCRF + } + if strings.TrimSpace(o.FFmpegPath) == "" { + o.FFmpegPath = "ffmpeg" + } + return o +} + +// ImageResult is compressed image output. +type ImageResult struct { + Data []byte + Width int + Height int + MIME string +} + +// CompressImage scales longest edge to ≤ Limits.ImageMaxEdge and re-encodes as JPEG (PNG in → JPEG out unless already small PNG kept as PNG when no resize needed and source is PNG). +func CompressImage(src []byte, opt Options) (*ImageResult, error) { + opt = opt.normalize() + if len(src) == 0 { + return nil, fmt.Errorf("media: empty image") + } + img, format, err := image.Decode(bytes.NewReader(src)) + if err != nil { + return nil, fmt.Errorf("media: decode image: %w", err) + } + b := img.Bounds() + w, h := b.Dx(), b.Dy() + maxEdge := opt.Limits.ImageMaxEdge + if !pkgmedia.NeedsResize(w, h, maxEdge) && int64(len(src)) <= opt.Limits.ImageMaxBytes { + mime := mimeForFormat(format) + return &ImageResult{Data: src, Width: w, Height: h, MIME: mime}, nil + } + nw, nh := scaleToMax(w, h, maxEdge) + dst := image.NewRGBA(image.Rect(0, 0, nw, nh)) + draw.CatmullRom.Scale(dst, dst.Bounds(), img, b, draw.Over, nil) + var buf bytes.Buffer + if err := jpeg.Encode(&buf, dst, &jpeg.Options{Quality: opt.JPEGQuality}); err != nil { + return nil, fmt.Errorf("media: encode jpeg: %w", err) + } + out := buf.Bytes() + if int64(len(out)) > opt.Limits.ImageMaxBytes { + return nil, fmt.Errorf("media: compressed image still too large (%d > %d)", len(out), opt.Limits.ImageMaxBytes) + } + return &ImageResult{Data: out, Width: nw, Height: nh, MIME: "image/jpeg"}, nil +} + +// CompressAvatar scales to ≤ AvatarMaxEdge and JPEG-encodes. +func CompressAvatar(src []byte, opt Options) (*ImageResult, error) { + opt = opt.normalize() + lim := opt.Limits + lim.ImageMaxEdge = lim.AvatarMaxEdge + lim.ImageMaxBytes = lim.AvatarMaxBytes + opt.Limits = lim + return CompressImage(src, opt) +} + +func scaleToMax(w, h, maxEdge int) (int, int) { + if maxEdge <= 0 || !pkgmedia.NeedsResize(w, h, maxEdge) { + return w, h + } + edge := pkgmedia.LongestEdge(w, h) + nw := w * maxEdge / edge + nh := h * maxEdge / edge + if nw < 1 { + nw = 1 + } + if nh < 1 { + nh = 1 + } + return nw, nh +} + +func mimeForFormat(format string) string { + switch strings.ToLower(format) { + case "png": + return "image/png" + case "gif": + return "image/gif" + default: + return "image/jpeg" + } +} + +// EncodePNG is a test helper / bot helper for synthetic images. +func EncodePNG(img image.Image) ([]byte, error) { + var buf bytes.Buffer + if err := png.Encode(&buf, img); err != nil { + return nil, err + } + return buf.Bytes(), nil +} diff --git a/internal/infrastructure/media/image_test.go b/internal/infrastructure/media/image_test.go new file mode 100644 index 0000000..f37e117 --- /dev/null +++ b/internal/infrastructure/media/image_test.go @@ -0,0 +1,71 @@ +package media + +import ( + "image" + "image/color" + "testing" + + pkgmedia "spark-server/internal/pkg/media" +) + +func TestCompressImageDownscales(t *testing.T) { + img := image.NewRGBA(image.Rect(0, 0, 2000, 1000)) + for y := 0; y < 1000; y++ { + for x := 0; x < 2000; x++ { + img.Set(x, y, color.RGBA{R: uint8(x % 255), G: uint8(y % 255), B: 40, A: 255}) + } + } + src, err := EncodePNG(img) + if err != nil { + t.Fatal(err) + } + res, err := CompressImage(src, Options{Limits: pkgmedia.DefaultLimits(), JPEGQuality: 80}) + if err != nil { + t.Fatal(err) + } + if pkgmedia.LongestEdge(res.Width, res.Height) > pkgmedia.DefaultImageMaxEdge { + t.Fatalf("edge %dx%d still over 1080", res.Width, res.Height) + } + if res.MIME != "image/jpeg" { + t.Fatalf("mime %s", res.MIME) + } + small := image.NewRGBA(image.Rect(0, 0, 100, 80)) + smallSrc, err := EncodePNG(small) + if err != nil { + t.Fatal(err) + } + keep, err := CompressImage(smallSrc, Options{Limits: pkgmedia.DefaultLimits()}) + if err != nil { + t.Fatal(err) + } + if keep.Width != 100 || keep.Height != 80 { + t.Fatalf("should keep small image dims, got %dx%d", keep.Width, keep.Height) + } +} + +func TestCompressAvatar(t *testing.T) { + img := image.NewRGBA(image.Rect(0, 0, 500, 500)) + src, err := EncodePNG(img) + if err != nil { + t.Fatal(err) + } + res, err := CompressAvatar(src, Options{Limits: pkgmedia.DefaultLimits()}) + if err != nil { + t.Fatal(err) + } + if pkgmedia.LongestEdge(res.Width, res.Height) > pkgmedia.DefaultAvatarMaxEdge { + t.Fatalf("avatar %dx%d", res.Width, res.Height) + } +} + +func TestPrepareBotImage(t *testing.T) { + img := image.NewRGBA(image.Rect(0, 0, 1600, 900)) + src, err := EncodePNG(img) + if err != nil { + t.Fatal(err) + } + a, err := PrepareBotImage(src, Options{Limits: pkgmedia.DefaultLimits()}) + if err != nil || a == nil || a.Size <= 0 { + t.Fatalf("%+v %v", a, err) + } +} diff --git a/internal/infrastructure/media/video.go b/internal/infrastructure/media/video.go new file mode 100644 index 0000000..da20cd1 --- /dev/null +++ b/internal/infrastructure/media/video.go @@ -0,0 +1,148 @@ +package media + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" + + pkgmedia "spark-server/internal/pkg/media" +) + +// VideoResult is compressed video output (H.264 MP4). +type VideoResult struct { + Data []byte + Width int + Height int + MIME string +} + +// CompressVideoBytes writes src to a temp file, runs ffmpeg scale+libx264, returns MP4 bytes. +// Requires ffmpeg on PATH (or Options.FFmpegPath). Returns error if ffmpeg is missing. +func CompressVideoBytes(ctx context.Context, src []byte, opt Options) (*VideoResult, error) { + opt = opt.normalize() + if len(src) == 0 { + return nil, fmt.Errorf("media: empty video") + } + if int64(len(src)) > opt.Limits.VideoMaxBytes*2 { + // refuse absurd remote downloads before temp write + return nil, fmt.Errorf("media: video source too large") + } + dir, err := os.MkdirTemp("", "spark-media-*") + if err != nil { + return nil, err + } + defer func() { _ = os.RemoveAll(dir) }() + inPath := filepath.Join(dir, "in.bin") + outPath := filepath.Join(dir, "out.mp4") + if err := os.WriteFile(inPath, src, 0o600); err != nil { + return nil, err + } + w, h, err := probeVideoSize(ctx, opt.FFmpegPath, inPath) + if err != nil { + // ffprobe may be absent; fall through to always re-encode + w, h = 0, 0 + } + if w > 0 && h > 0 && !pkgmedia.NeedsResize(w, h, opt.Limits.VideoMaxEdge) && int64(len(src)) <= opt.Limits.VideoMaxBytes { + return &VideoResult{Data: src, Width: w, Height: h, MIME: "video/mp4"}, nil + } + if err := runFFmpegCompress(ctx, opt, inPath, outPath); err != nil { + return nil, err + } + out, err := os.ReadFile(outPath) + if err != nil { + return nil, err + } + if int64(len(out)) > opt.Limits.VideoMaxBytes { + return nil, fmt.Errorf("media: compressed video still too large (%d > %d)", len(out), opt.Limits.VideoMaxBytes) + } + ow, oh, _ := probeVideoSize(ctx, opt.FFmpegPath, outPath) + if ow <= 0 { + ow, oh = w, h + } + return &VideoResult{Data: out, Width: ow, Height: oh, MIME: "video/mp4"}, nil +} + +func runFFmpegCompress(ctx context.Context, opt Options, inPath, outPath string) error { + if ctx == nil { + ctx = context.Background() + } + if _, ok := ctx.Deadline(); !ok { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, 3*time.Minute) + defer cancel() + } + maxEdge := opt.Limits.VideoMaxEdge + // Longest side ≤ maxEdge; force_original_aspect_ratio keeps aspect, even dims for H.264. + vf := fmt.Sprintf("scale=%d:%d:force_original_aspect_ratio=decrease,pad=ceil(iw/2)*2:ceil(ih/2)*2", maxEdge, maxEdge) + args := []string{ + "-y", "-hide_banner", "-loglevel", "error", + "-i", inPath, + "-vf", vf, + "-c:v", "libx264", "-crf", strconv.Itoa(opt.VideoCRF), "-preset", "veryfast", + "-c:a", "aac", "-b:a", "128k", + "-movflags", "+faststart", + outPath, + } + cmd := exec.CommandContext(ctx, opt.FFmpegPath, args...) + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + msg := strings.TrimSpace(stderr.String()) + if msg == "" { + msg = err.Error() + } + if isExecNotFound(err) { + return fmt.Errorf("media: ffmpeg not found (install ffmpeg for bot video compress): %w", err) + } + return fmt.Errorf("media: ffmpeg: %s", msg) + } + return nil +} + +func probeVideoSize(ctx context.Context, ffmpegPath, path string) (w, h int, err error) { + // Prefer ffprobe beside ffmpeg; fall back to ffmpeg -i parse. + probe := strings.Replace(ffmpegPath, "ffmpeg", "ffprobe", 1) + if probe == ffmpegPath { + probe = "ffprobe" + } + cmd := exec.CommandContext(ctx, probe, + "-v", "error", "-select_streams", "v:0", + "-show_entries", "stream=width,height", "-of", "csv=p=0:s=x", path, + ) + out, err := cmd.Output() + if err != nil { + return 0, 0, err + } + parts := strings.Split(strings.TrimSpace(string(out)), "x") + if len(parts) != 2 { + return 0, 0, fmt.Errorf("media: bad ffprobe output") + } + w, _ = strconv.Atoi(parts[0]) + h, _ = strconv.Atoi(parts[1]) + if w <= 0 || h <= 0 { + return 0, 0, fmt.Errorf("media: invalid video size") + } + return w, h, nil +} + +func isExecNotFound(err error) bool { + if err == nil { + return false + } + return strings.Contains(err.Error(), "executable file not found") || + strings.Contains(err.Error(), "no such file") || + os.IsNotExist(err) +} + +// FFmpegAvailable reports whether the configured ffmpeg binary can be executed. +func FFmpegAvailable(opt Options) bool { + opt = opt.normalize() + path, err := exec.LookPath(opt.FFmpegPath) + return err == nil && path != "" +} diff --git a/internal/infrastructure/objectstore/ops.go b/internal/infrastructure/objectstore/ops.go index 293e1e7..f675fc6 100644 --- a/internal/infrastructure/objectstore/ops.go +++ b/internal/infrastructure/objectstore/ops.go @@ -21,10 +21,10 @@ import ( const ( maxGetBytes = 32 << 20 maxPutBytes = 4 << 20 - maxPutLarge = 50 << 20 // admin music audio etc. + maxPutLarge = 100 << 20 // admin music / bot video after compress (≤100 MiB) ) -// PutLarge writes an object up to maxBytes (capped at 50 MiB). +// PutLarge writes an object up to maxBytes (capped at 100 MiB). func (s *Store) PutLarge(ctx context.Context, key string, body []byte, contentType string, maxBytes int) error { if !s.Ready() { return errs.ErrAvailable diff --git a/internal/module/meta/logic/enums_wire.go b/internal/module/meta/logic/enums_wire.go index 0b3383d..ce4ef7a 100644 --- a/internal/module/meta/logic/enums_wire.go +++ b/internal/module/meta/logic/enums_wire.go @@ -38,6 +38,7 @@ func envelopeCode(lang i18n.Lang) []entity.Item { code(lang, "frame_too_large", errs.ErrFrameTooLarge), code(lang, "invalid_frame", errs.ErrInvalidFrame), code(lang, "invalid_proxy_header", errs.ErrProxyHeader), + code(lang, "media_oversized", errs.ErrMediaOversized), code(lang, "unauthorized", errs.ErrUnauthorized), code(lang, "not_authenticated", errs.ErrNotAuthenticated), code(lang, "verify_code_invalid", errs.ErrVerifyCodeInvalid), diff --git a/internal/module/moment/logic/media_check.go b/internal/module/moment/logic/media_check.go new file mode 100644 index 0000000..b78ea49 --- /dev/null +++ b/internal/module/moment/logic/media_check.go @@ -0,0 +1,55 @@ +package logic + +import ( + "context" + + "spark-server/internal/module/moment/entity" + "spark-server/internal/pkg/errs" + "spark-server/internal/pkg/i18n" + "spark-server/internal/pkg/media" + "spark-server/internal/pkg/reqctx" + + "go.uber.org/zap" +) + +// checkMediaMeta validates client-declared w/h/size on user publish. +func (s *Service) checkMediaMeta(ctx context.Context, images []*entity.Image, video *entity.Video) error { + if s == nil || !s.media.Enabled { + return nil + } + var v *media.Violation + if video != nil { + v = s.media.CheckPublishVideo(&media.VideoMeta{ + Width: video.Width, Height: video.Height, Size: video.Size, DurationMS: video.DurationMS, + }) + } else { + metas := make([]media.ImageMeta, 0, len(images)) + for _, img := range images { + if img == nil { + continue + } + metas = append(metas, media.ImageMeta{Width: img.Width, Height: img.Height, Size: img.Size}) + } + v = s.media.CheckPublishImages(metas) + } + if v == nil { + return nil + } + log := s.log + if log == nil { + log = zap.NewNop() + } + log.Warn("moment media oversize", + zap.String("kind", string(v.Kind)), + zap.String("reason", v.Reason), + zap.Int("w", v.Width), + zap.Int("h", v.Height), + zap.Int("max_edge", v.MaxEdge), + zap.Int64("size", v.Size), + zap.Int64("max_size", v.MaxSize), + ) + if !s.media.UserReject { + return nil + } + return errs.ErrMediaOversized.WithMsg(i18n.T(reqctx.Lang(ctx), i18n.KeyMediaOversized, nil)) +} diff --git a/internal/module/moment/logic/publish.go b/internal/module/moment/logic/publish.go index 8334d32..5abd042 100644 --- a/internal/module/moment/logic/publish.go +++ b/internal/module/moment/logic/publish.go @@ -40,6 +40,9 @@ func (s *Service) Publish(ctx context.Context, userID int64, in *PublishInput) ( if err != nil { return nil, err } + if err := s.checkMediaMeta(ctx, in.Images, in.Video); err != nil { + return nil, err + } raw, err := content.Marshal() if err != nil { return nil, errs.ErrInvalidArgument.WithMsg(i18n.T(reqctx.Lang(ctx), i18n.KeyMomentContentInvalid, nil)) diff --git a/internal/module/moment/logic/service.go b/internal/module/moment/logic/service.go index f30637b..892272b 100644 --- a/internal/module/moment/logic/service.go +++ b/internal/module/moment/logic/service.go @@ -6,6 +6,9 @@ import ( friendentity "spark-server/internal/module/friend/entity" "spark-server/internal/module/moment/entity" userentity "spark-server/internal/module/user/entity" + "spark-server/internal/pkg/media" + + "go.uber.org/zap" ) // Repository persists Moments. Model implements it. @@ -71,10 +74,23 @@ type Service struct { objects Objects ids IDGen burst *burst + media media.Policy + log *zap.Logger } func NewService(repo Repository, friends Friends, users Users, objects Objects, ids IDGen) *Service { - return &Service{repo: repo, friends: friends, users: users, objects: objects, ids: ids, burst: newBurst()} + return &Service{repo: repo, friends: friends, users: users, objects: objects, ids: ids, burst: newBurst(), log: zap.NewNop()} +} + +// ConfigureMedia sets user-path media detection (client compresses; server checks declared meta). +func (s *Service) ConfigureMedia(p media.Policy, log *zap.Logger) { + if s == nil { + return + } + s.media = p + if log != nil { + s.log = log + } } func (s *Service) ready() bool { diff --git a/internal/module/square/entity/post.go b/internal/module/square/entity/post.go index e77e982..510f3fe 100644 --- a/internal/module/square/entity/post.go +++ b/internal/module/square/entity/post.go @@ -11,11 +11,25 @@ type Post struct { Visibility Visibility AuditStatus AuditStatus Status Status + Source Source + SourceURL string + ExpiresAtMS int64 // 0 = never expires LikeCount int CommentCount int CreatedAtMS int64 } +// Expired reports whether a bot post has passed expires_at (ms). +func (p *Post) Expired(nowMS int64) bool { + if p == nil || p.ExpiresAtMS <= 0 { + return false + } + if nowMS <= 0 { + return false + } + return nowMS >= p.ExpiresAtMS +} + // POI is an optional location pin. type POI struct { Lat float64 diff --git a/internal/module/square/entity/source.go b/internal/module/square/entity/source.go new file mode 100644 index 0000000..3a98094 --- /dev/null +++ b/internal/module/square/entity/source.go @@ -0,0 +1,18 @@ +package entity + +// Source is square_posts.source. +type Source int16 + +const ( + SourceUser Source = 0 + SourceBotRSS Source = 1 +) + +func (s Source) Valid() bool { + return s == SourceUser || s == SourceBotRSS +} + +// IsBot reports whether the post was published by the media bot worker. +func (s Source) IsBot() bool { + return s == SourceBotRSS +} diff --git a/internal/module/square/logic/bot_publish.go b/internal/module/square/logic/bot_publish.go new file mode 100644 index 0000000..babd161 --- /dev/null +++ b/internal/module/square/logic/bot_publish.go @@ -0,0 +1,83 @@ +package logic + +import ( + "context" + "strings" + "time" + "unicode/utf8" + + "spark-server/internal/module/square/entity" + "spark-server/internal/pkg/errs" + "spark-server/internal/pkg/i18n" + "spark-server/internal/pkg/reqctx" +) + +// BotPublishInput is an internal publish from the media bot worker (no rate limit). +type BotPublishInput struct { + Text string + SourceURL string + TTL time.Duration +} + +// PublishBotMedia publishes a public text post as a bot user with TTL and URL dedupe. +func (s *Service) PublishBotMedia(ctx context.Context, userID int64, in *BotPublishInput) (*View, error) { + if !s.ready() || userID <= 0 { + return nil, errs.ErrUnauthorized + } + if in == nil { + return nil, errs.ErrInvalidArgument + } + text := strings.TrimSpace(in.Text) + url := strings.TrimSpace(in.SourceURL) + if text == "" || url == "" { + return nil, errs.ErrInvalidArgument.WithMsg(i18n.T(reqctx.Lang(ctx), i18n.KeySquareEmpty, nil)) + } + if utf8.RuneCountInString(text) > entity.MaxTextRunes { + return nil, errs.ErrInvalidArgument.WithMsg(i18n.T(reqctx.Lang(ctx), i18n.KeySquareTextInvalid, nil)) + } + ttl := in.TTL + if ttl <= 0 { + ttl = 15 * 24 * time.Hour + } + content := &entity.Content{} + raw, err := content.Marshal() + if err != nil { + return nil, errs.ErrInvalidArgument.WithMsg(i18n.T(reqctx.Lang(ctx), i18n.KeySquareContentInvalid, nil)) + } + id, err := s.ids.NextID() + if err != nil { + return nil, err + } + now := time.Now() + post := &entity.Post{ + ID: id, UserID: userID, Kind: entity.KindText, Text: text, Content: raw, + Visibility: entity.VisibilityPublic, AuditStatus: entity.AuditApproved, + Status: entity.StatusNormal, Source: entity.SourceBotRSS, SourceURL: url, + ExpiresAtMS: now.Add(ttl).UnixMilli(), CreatedAtMS: now.UnixMilli(), + } + if err := s.repo.CreateBotPost(ctx, post); err != nil { + return nil, err + } + return s.viewOf(ctx, userID, post, true) +} + +// ExpireBotPosts soft-deletes expired bot posts and best-effort deletes media. +func (s *Service) ExpireBotPosts(ctx context.Context, now time.Time, limit int) (int, error) { + if !s.ready() { + return 0, nil + } + if now.IsZero() { + now = time.Now() + } + if limit <= 0 { + limit = 200 + } + posts, err := s.repo.SoftDeleteExpiredBotPosts(ctx, now.UnixMilli(), limit) + if err != nil { + return 0, err + } + for _, post := range posts { + s.deleteMedia(ctx, post) + } + return len(posts), nil +} diff --git a/internal/module/square/logic/media_check.go b/internal/module/square/logic/media_check.go new file mode 100644 index 0000000..f66fa84 --- /dev/null +++ b/internal/module/square/logic/media_check.go @@ -0,0 +1,56 @@ +package logic + +import ( + "context" + + "spark-server/internal/module/square/entity" + "spark-server/internal/pkg/errs" + "spark-server/internal/pkg/i18n" + "spark-server/internal/pkg/media" + "spark-server/internal/pkg/reqctx" + + "go.uber.org/zap" +) + +// checkMediaMeta validates client-declared w/h/size on user publish. +// Default: log only; media.user_reject=true returns ErrMediaOversized (40005). +func (s *Service) checkMediaMeta(ctx context.Context, images []*entity.Image, video *entity.Video) error { + if s == nil || !s.media.Enabled { + return nil + } + var v *media.Violation + if video != nil { + v = s.media.CheckPublishVideo(&media.VideoMeta{ + Width: video.Width, Height: video.Height, Size: video.Size, DurationMS: video.DurationMS, + }) + } else { + metas := make([]media.ImageMeta, 0, len(images)) + for _, img := range images { + if img == nil { + continue + } + metas = append(metas, media.ImageMeta{Width: img.Width, Height: img.Height, Size: img.Size}) + } + v = s.media.CheckPublishImages(metas) + } + if v == nil { + return nil + } + log := s.log + if log == nil { + log = zap.NewNop() + } + log.Warn("square media oversize", + zap.String("kind", string(v.Kind)), + zap.String("reason", v.Reason), + zap.Int("w", v.Width), + zap.Int("h", v.Height), + zap.Int("max_edge", v.MaxEdge), + zap.Int64("size", v.Size), + zap.Int64("max_size", v.MaxSize), + ) + if !s.media.UserReject { + return nil + } + return errs.ErrMediaOversized.WithMsg(i18n.T(reqctx.Lang(ctx), i18n.KeyMediaOversized, nil)) +} diff --git a/internal/module/square/logic/publish.go b/internal/module/square/logic/publish.go index 069373f..132ae50 100644 --- a/internal/module/square/logic/publish.go +++ b/internal/module/square/logic/publish.go @@ -40,6 +40,9 @@ func (s *Service) Publish(ctx context.Context, userID int64, in *PublishInput) ( if err != nil { return nil, err } + if err := s.checkMediaMeta(ctx, in.Images, in.Video); err != nil { + return nil, err + } raw, err := content.Marshal() if err != nil { return nil, errs.ErrInvalidArgument.WithMsg(i18n.T(reqctx.Lang(ctx), i18n.KeySquareContentInvalid, nil)) diff --git a/internal/module/square/logic/service.go b/internal/module/square/logic/service.go index 372608d..f52d083 100644 --- a/internal/module/square/logic/service.go +++ b/internal/module/square/logic/service.go @@ -6,6 +6,9 @@ import ( friendentity "spark-server/internal/module/friend/entity" "spark-server/internal/module/square/entity" userentity "spark-server/internal/module/user/entity" + "spark-server/internal/pkg/media" + + "go.uber.org/zap" ) // Repository persists square data. Model implements it. @@ -64,10 +67,23 @@ type Service struct { objects Objects ids IDGen burst *burst + media media.Policy + log *zap.Logger } func NewService(repo Repository, friends Friends, users Users, objects Objects, ids IDGen) *Service { - return &Service{repo: repo, friends: friends, users: users, objects: objects, ids: ids, burst: newBurst()} + return &Service{repo: repo, friends: friends, users: users, objects: objects, ids: ids, burst: newBurst(), log: zap.NewNop()} +} + +// ConfigureMedia sets user-path media detection (bots compress via squarebot MediaCompressor). +func (s *Service) ConfigureMedia(p media.Policy, log *zap.Logger) { + if s == nil { + return + } + s.media = p + if log != nil { + s.log = log + } } func (s *Service) ready() bool { diff --git a/internal/module/square/logic/visible.go b/internal/module/square/logic/visible.go index d188c26..4703505 100644 --- a/internal/module/square/logic/visible.go +++ b/internal/module/square/logic/visible.go @@ -2,6 +2,7 @@ package logic import ( "context" + "time" "spark-server/internal/module/square/entity" ) @@ -16,6 +17,9 @@ func (v *visCtx) canSee(p *entity.Post) bool { if p == nil || p.Status != entity.StatusNormal || p.AuditStatus != entity.AuditApproved { return false } + if p.Expired(time.Now().UnixMilli()) { + return false + } if p.UserID != v.viewer && v.blocked[p.UserID] { return false } diff --git a/internal/module/square/model/bot.go b/internal/module/square/model/bot.go new file mode 100644 index 0000000..46f5d54 --- /dev/null +++ b/internal/module/square/model/bot.go @@ -0,0 +1,103 @@ +package model + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "strings" + "time" + + "gorm.io/gorm" + "gorm.io/gorm/clause" + + "spark-server/internal/module/square/entity" + "spark-server/internal/pkg/errs" +) + +// CreateBotPost inserts a bot RSS post and claims its source URL atomically. +func (m *Repo) CreateBotPost(ctx context.Context, post *entity.Post) error { + if post == nil || post.ID <= 0 || post.UserID <= 0 { + return errs.ErrInvalidArgument + } + url := strings.TrimSpace(post.SourceURL) + if url == "" { + return errs.ErrInvalidArgument + } + post.SourceURL = url + if post.Source == 0 { + post.Source = entity.SourceBotRSS + } + db, err := m.session(ctx) + if err != nil { + return err + } + row := postFrom(post) + if row.CreatedAt.IsZero() { + row.CreatedAt = time.Now() + } + hash := urlHash(url) + return db.Transaction(func(tx *gorm.DB) error { + link := BotLink{URLHash: hash, URL: url, PostID: post.ID, CreatedAt: row.CreatedAt} + res := tx.Clauses(clause.OnConflict{DoNothing: true}).Create(&link) + if res.Error != nil { + return mapDBErr(res.Error) + } + if res.RowsAffected == 0 { + return errs.ErrDuplicate + } + if err := tx.Create(row).Error; err != nil { + return mapDBErr(err) + } + return nil + }) +} + +// SoftDeleteExpiredBotPosts marks expired bot posts deleted; returns deleted rows for media GC. +func (m *Repo) SoftDeleteExpiredBotPosts(ctx context.Context, nowMS int64, limit int) ([]*entity.Post, error) { + if limit <= 0 { + limit = 100 + } + if nowMS <= 0 { + nowMS = time.Now().UnixMilli() + } + db, err := m.session(ctx) + if err != nil { + return nil, err + } + deadline := time.UnixMilli(nowMS) + var rows []Post + err = db.Transaction(func(tx *gorm.DB) error { + q := tx.Where( + "source = ? AND status = ? AND expires_at IS NOT NULL AND expires_at <= ?", + entity.SourceBotRSS, entity.StatusNormal, deadline, + ).Order("expires_at ASC").Limit(limit) + if err := q.Find(&rows).Error; err != nil { + return mapDBErr(err) + } + if len(rows) == 0 { + return nil + } + ids := make([]int64, 0, len(rows)) + for i := range rows { + ids = append(ids, rows[i].ID) + } + res := tx.Model(&Post{}).Where("id IN ? AND status = ?", ids, entity.StatusNormal). + Update("status", entity.StatusDeleted) + if res.Error != nil { + return mapDBErr(res.Error) + } + for i := range rows { + rows[i].Status = int16(entity.StatusDeleted) + } + return nil + }) + if err != nil { + return nil, err + } + return collectPosts(rows), nil +} + +func urlHash(url string) string { + sum := sha256.Sum256([]byte(strings.TrimSpace(url))) + return hex.EncodeToString(sum[:]) +} diff --git a/internal/module/square/model/memory.go b/internal/module/square/model/memory.go index d365804..4197f0e 100644 --- a/internal/module/square/model/memory.go +++ b/internal/module/square/model/memory.go @@ -4,6 +4,7 @@ import ( "context" "sort" "sync" + "time" "spark-server/internal/module/square/entity" "spark-server/internal/pkg/errs" @@ -23,6 +24,7 @@ type MemoryRepo struct { comments map[int64]*entity.Comment notices map[int64]*entity.Notice noticeState map[int64]*entity.NoticeState + botLinks map[string]int64 // url -> post_id } func NewMemoryRepo() *MemoryRepo { @@ -33,6 +35,7 @@ func NewMemoryRepo() *MemoryRepo { comments: make(map[int64]*entity.Comment), notices: make(map[int64]*entity.Notice), noticeState: make(map[int64]*entity.NoticeState), + botLinks: make(map[string]int64), } } @@ -50,11 +53,15 @@ func (m *MemoryRepo) ListByAuthors(_ context.Context, authorIDs []int64, cursor m.mu.Lock() defer m.mu.Unlock() allow := toSet(authorIDs) + nowMS := time.Now().UnixMilli() out := make([]*entity.Post, 0) for _, row := range m.posts { if row.Status != entity.StatusNormal || !allow[row.UserID] { continue } + if row.Expired(nowMS) { + continue + } if cursor > 0 && row.ID >= cursor { continue } @@ -133,6 +140,7 @@ func (m *MemoryRepo) GetComment(_ context.Context, id int64) (*entity.Comment, e func (m *MemoryRepo) ListPublicFeed(_ context.Context, cursor int64, limit int) ([]*entity.Post, error) { m.mu.Lock() defer m.mu.Unlock() + nowMS := time.Now().UnixMilli() out := make([]*entity.Post, 0) for _, row := range m.posts { if row.Status != entity.StatusNormal || row.AuditStatus != entity.AuditApproved { @@ -141,6 +149,9 @@ func (m *MemoryRepo) ListPublicFeed(_ context.Context, cursor int64, limit int) if row.Visibility != entity.VisibilityPublic && row.Visibility != entity.VisibilityExclude { continue } + if row.Expired(nowMS) { + continue + } if cursor > 0 && row.ID >= cursor { continue } diff --git a/internal/module/square/model/memory_write.go b/internal/module/square/model/memory_write.go index f4309e8..59eecea 100644 --- a/internal/module/square/model/memory_write.go +++ b/internal/module/square/model/memory_write.go @@ -2,6 +2,7 @@ package model import ( "context" + "strings" "time" "spark-server/internal/module/square/entity" @@ -24,6 +25,57 @@ func (m *MemoryRepo) CreatePost(_ context.Context, post *entity.Post, audience [ return nil } +func (m *MemoryRepo) CreateBotPost(_ context.Context, post *entity.Post) error { + if post == nil || post.ID <= 0 || post.UserID <= 0 { + return errs.ErrInvalidArgument + } + url := strings.TrimSpace(post.SourceURL) + if url == "" { + return errs.ErrInvalidArgument + } + post.SourceURL = url + if post.Source == 0 { + post.Source = entity.SourceBotRSS + } + m.mu.Lock() + defer m.mu.Unlock() + if _, ok := m.botLinks[url]; ok { + return errs.ErrDuplicate + } + if _, ok := m.posts[post.ID]; ok { + return errs.ErrDuplicate + } + m.botLinks[url] = post.ID + m.posts[post.ID] = clonePost(post) + return nil +} + +func (m *MemoryRepo) SoftDeleteExpiredBotPosts(_ context.Context, nowMS int64, limit int) ([]*entity.Post, error) { + if limit <= 0 { + limit = 100 + } + if nowMS <= 0 { + nowMS = time.Now().UnixMilli() + } + m.mu.Lock() + defer m.mu.Unlock() + out := make([]*entity.Post, 0) + for _, row := range m.posts { + if row.Status != entity.StatusNormal || !row.Source.IsBot() { + continue + } + if !row.Expired(nowMS) { + continue + } + row.Status = entity.StatusDeleted + out = append(out, clonePost(row)) + if len(out) >= limit { + break + } + } + return out, nil +} + func (m *MemoryRepo) SoftDeletePost(_ context.Context, userID, postID int64) (*entity.Post, error) { m.mu.Lock() defer m.mu.Unlock() diff --git a/internal/module/square/model/post.go b/internal/module/square/model/post.go index 60483bc..d242638 100644 --- a/internal/module/square/model/post.go +++ b/internal/module/square/model/post.go @@ -22,6 +22,9 @@ type Post struct { AuditNote string `gorm:"column:audit_note;comment:拒绝原因"` AuditedAt *time.Time `gorm:"column:audited_at;comment:审核时间"` Status int16 `gorm:"column:status;comment:0正常 1已删"` + Source int16 `gorm:"column:source;comment:0用户 1机器人海外媒体RSS"` + SourceURL string `gorm:"column:source_url;comment:原文 URL"` + ExpiresAt *time.Time `gorm:"column:expires_at;comment:机器人帖过期时间"` LikeCount int `gorm:"column:like_count;comment:点赞数"` CommentCount int `gorm:"column:comment_count;comment:评论数"` CreatedAt time.Time `gorm:"column:created_at;comment:发帖时间"` @@ -37,8 +40,11 @@ func (r *Post) Entity() *entity.Post { ID: r.ID, UserID: r.UserID, Kind: entity.Kind(r.Kind), Text: r.Text, Content: r.Content, POI: entity.POI{Lat: r.POILat, Lng: r.POILng, Name: r.POIName, Address: r.POIAddress}, Visibility: entity.Visibility(r.Visibility), AuditStatus: entity.AuditStatus(r.AuditStatus), - Status: entity.Status(r.Status), LikeCount: r.LikeCount, CommentCount: r.CommentCount, - CreatedAtMS: r.CreatedAt.UnixMilli(), + Status: entity.Status(r.Status), Source: entity.Source(r.Source), SourceURL: r.SourceURL, + LikeCount: r.LikeCount, CommentCount: r.CommentCount, CreatedAtMS: r.CreatedAt.UnixMilli(), + } + if r.ExpiresAt != nil { + p.ExpiresAtMS = r.ExpiresAt.UnixMilli() } return p } @@ -51,11 +57,15 @@ func postFrom(p *entity.Post) *Post { ID: p.ID, UserID: p.UserID, Kind: int16(p.Kind), Text: p.Text, Content: p.Content, POILat: p.POI.Lat, POILng: p.POI.Lng, POIName: p.POI.Name, POIAddress: p.POI.Address, Visibility: int16(p.Visibility), AuditStatus: int16(p.AuditStatus), Status: int16(p.Status), - LikeCount: p.LikeCount, CommentCount: p.CommentCount, + Source: int16(p.Source), SourceURL: p.SourceURL, LikeCount: p.LikeCount, CommentCount: p.CommentCount, } if p.CreatedAtMS > 0 { row.CreatedAt = time.UnixMilli(p.CreatedAtMS) } + if p.ExpiresAtMS > 0 { + t := time.UnixMilli(p.ExpiresAtMS) + row.ExpiresAt = &t + } if p.AuditStatus == entity.AuditApproved { t := row.CreatedAt if t.IsZero() { @@ -66,6 +76,16 @@ func postFrom(p *entity.Post) *Post { return row } +// BotLink maps t_square_bot_links (RSS URL 去重). +type BotLink struct { + URLHash string `gorm:"column:url_hash;primaryKey;comment:URL SHA-256 hex"` + URL string `gorm:"column:url;comment:原文 URL"` + PostID int64 `gorm:"column:post_id;comment:帖 id"` + CreatedAt time.Time `gorm:"column:created_at;comment:入库时间"` +} + +func (BotLink) TableName() string { return "t_square_bot_links" } + // Audience maps square_audience. type Audience struct { PostID int64 `gorm:"column:post_id;primaryKey;comment:帖 id"` diff --git a/internal/module/square/model/query.go b/internal/module/square/model/query.go index 98948f5..498f4a0 100644 --- a/internal/module/square/model/query.go +++ b/internal/module/square/model/query.go @@ -3,12 +3,16 @@ package model import ( "context" "errors" + "time" "gorm.io/gorm" "spark-server/internal/module/square/entity" ) +// notExpiredSQL keeps user posts and unexpired bot posts. +const notExpiredSQL = "(expires_at IS NULL OR expires_at > ?)" + func (m *Repo) GetPost(ctx context.Context, id int64) (*entity.Post, error) { db, err := m.session(ctx) if err != nil { @@ -29,7 +33,7 @@ func (m *Repo) ListByAuthors(ctx context.Context, authorIDs []int64, cursor int6 if err != nil { return nil, err } - q := db.Where("user_id IN ? AND status = ?", authorIDs, entity.StatusNormal) + q := db.Where("user_id IN ? AND status = ? AND "+notExpiredSQL, authorIDs, entity.StatusNormal, time.Now()) if cursor > 0 { q = q.Where("id < ?", cursor) } @@ -50,9 +54,10 @@ func (m *Repo) ListPublicFeed(ctx context.Context, cursor int64, limit int) ([]* return nil, err } q := db.Where( - "audit_status = ? AND status = ? AND visibility IN ?", + "audit_status = ? AND status = ? AND visibility IN ? AND "+notExpiredSQL, entity.AuditApproved, entity.StatusNormal, []int16{int16(entity.VisibilityPublic), int16(entity.VisibilityExclude)}, + time.Now(), ) if cursor > 0 { q = q.Where("id < ?", cursor) diff --git a/internal/module/squarebot/logic/.keep b/internal/module/squarebot/logic/.keep new file mode 100644 index 0000000..e69de29 diff --git a/internal/module/squarebot/logic/doc.go b/internal/module/squarebot/logic/doc.go new file mode 100644 index 0000000..c993059 --- /dev/null +++ b/internal/module/squarebot/logic/doc.go @@ -0,0 +1,3 @@ +// Package logic is the square overseas-media bot worker (RSS → compress → square post). +// Media helpers live in media.go so the RSS poller can compress before objectstore Put. +package logic diff --git a/internal/module/squarebot/logic/media.go b/internal/module/squarebot/logic/media.go new file mode 100644 index 0000000..8153577 --- /dev/null +++ b/internal/module/squarebot/logic/media.go @@ -0,0 +1,104 @@ +package logic + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + "time" + + infmedia "spark-server/internal/infrastructure/media" + pkgmedia "spark-server/internal/pkg/media" +) + +const maxDownloadBytes = 80 << 20 // refuse absurd remote blobs before compress + +// MediaCompressor downloads remote media and compresses to App-aligned specs before objectstore write. +// Square bot RSS worker should call FetchImage / FetchVideo instead of raw HTTP+Put. +type MediaCompressor struct { + Opt infmedia.Options + Client *http.Client +} + +// NewMediaCompressor builds a compressor; ffmpegPath may be empty (defaults to "ffmpeg"). +func NewMediaCompressor(lim pkgmedia.Limits, jpegQuality, videoCRF int, ffmpegPath string) *MediaCompressor { + return &MediaCompressor{ + Opt: infmedia.OptionsFromLimits(lim, jpegQuality, videoCRF, ffmpegPath), + Client: &http.Client{ + Timeout: 60 * time.Second, + }, + } +} + +// FetchImage GETs url and compresses to ≤1080p JPEG/PNG. +func (c *MediaCompressor) FetchImage(ctx context.Context, url string) (*infmedia.BotAsset, error) { + raw, err := c.download(ctx, url) + if err != nil { + return nil, err + } + return infmedia.PrepareBotImage(raw, c.Opt) +} + +// FetchAvatar GETs url and compresses to ≤200×200. +func (c *MediaCompressor) FetchAvatar(ctx context.Context, url string) (*infmedia.BotAsset, error) { + raw, err := c.download(ctx, url) + if err != nil { + return nil, err + } + return infmedia.PrepareBotAvatar(raw, c.Opt) +} + +// FetchVideo GETs url and compresses to ≤720p (requires ffmpeg). +func (c *MediaCompressor) FetchVideo(ctx context.Context, url string) (*infmedia.BotAsset, error) { + raw, err := c.download(ctx, url) + if err != nil { + return nil, err + } + return infmedia.PrepareBotVideo(ctx, raw, c.Opt) +} + +// CompressBytesImage compresses already-downloaded image bytes (e.g. RSS enclosure already fetched). +func (c *MediaCompressor) CompressBytesImage(src []byte) (*infmedia.BotAsset, error) { + return infmedia.PrepareBotImage(src, c.Opt) +} + +// CompressBytesVideo compresses already-downloaded video bytes. +func (c *MediaCompressor) CompressBytesVideo(ctx context.Context, src []byte) (*infmedia.BotAsset, error) { + return infmedia.PrepareBotVideo(ctx, src, c.Opt) +} + +func (c *MediaCompressor) download(ctx context.Context, url string) ([]byte, error) { + url = strings.TrimSpace(url) + if url == "" { + return nil, fmt.Errorf("squarebot media: empty url") + } + client := c.Client + if client == nil { + client = http.DefaultClient + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", "SparkIM-SquareBot/1.0") + res, err := client.Do(req) + if err != nil { + return nil, err + } + defer func() { _ = res.Body.Close() }() + if res.StatusCode < 200 || res.StatusCode >= 300 { + return nil, fmt.Errorf("squarebot media: http %d", res.StatusCode) + } + data, err := io.ReadAll(io.LimitReader(res.Body, maxDownloadBytes+1)) + if err != nil { + return nil, err + } + if len(data) > maxDownloadBytes { + return nil, fmt.Errorf("squarebot media: download too large") + } + if len(data) == 0 { + return nil, fmt.Errorf("squarebot media: empty body") + } + return data, nil +} diff --git a/internal/module/squarebot/logic/rss.go b/internal/module/squarebot/logic/rss.go new file mode 100644 index 0000000..a5bb7e8 --- /dev/null +++ b/internal/module/squarebot/logic/rss.go @@ -0,0 +1,228 @@ +package logic + +import ( + "context" + "encoding/xml" + "fmt" + "html" + "io" + "net/http" + "net/url" + "regexp" + "strings" + "time" + "unicode/utf8" +) + +const ( + rssMaxBody = 2 << 20 + rssTimeout = 20 * time.Second + summaryRunes = 280 +) + +// FeedItem is one public RSS/Atom entry (title + summary + link). +type FeedItem struct { + Title string + Summary string + Link string + Source string +} + +type rssFeed struct { + Channel struct { + Title string `xml:"title"` + Items []rssItem `xml:"item"` + } `xml:"channel"` +} + +type rssItem struct { + Title string `xml:"title"` + Link string `xml:"link"` + Description string `xml:"description"` + GUID string `xml:"guid"` +} + +type atomFeed struct { + Title string `xml:"title"` + Entries []atomEntry `xml:"entry"` +} + +type atomEntry struct { + Title string `xml:"title"` + Summary string `xml:"summary"` + Content string `xml:"content"` + Links []atomLink `xml:"link"` + ID string `xml:"id"` +} + +type atomLink struct { + Href string `xml:"href,attr"` + Rel string `xml:"rel,attr"` +} + +var tagRe = regexp.MustCompile(`(?s)<[^>]*>`) + +// FetchFeed loads and parses a public RSS 2.0 or Atom URL. +func FetchFeed(ctx context.Context, client *http.Client, feedURL string) ([]FeedItem, error) { + feedURL = strings.TrimSpace(feedURL) + if feedURL == "" { + return nil, fmt.Errorf("empty feed url") + } + if client == nil { + client = &http.Client{Timeout: rssTimeout} + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, feedURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", "SparkIM-SquareBot/1.0 (+https://spark.wyrs365.com)") + req.Header.Set("Accept", "application/rss+xml, application/atom+xml, application/xml, text/xml, */*") + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, fmt.Errorf("feed status %d", resp.StatusCode) + } + body, err := io.ReadAll(io.LimitReader(resp.Body, rssMaxBody)) + if err != nil { + return nil, err + } + return parseFeed(body, feedURL) +} + +func parseFeed(body []byte, feedURL string) ([]FeedItem, error) { + var rss rssFeed + if err := xml.Unmarshal(body, &rss); err == nil && len(rss.Channel.Items) > 0 { + src := strings.TrimSpace(rss.Channel.Title) + out := make([]FeedItem, 0, len(rss.Channel.Items)) + for _, it := range rss.Channel.Items { + link := firstNonEmpty(strings.TrimSpace(it.Link), strings.TrimSpace(it.GUID)) + item := FeedItem{ + Title: cleanText(it.Title), + Summary: cleanText(it.Description), + Link: normalizeURL(link), + Source: src, + } + if item.Title == "" || item.Link == "" { + continue + } + out = append(out, item) + } + if len(out) > 0 { + return out, nil + } + } + var atom atomFeed + if err := xml.Unmarshal(body, &atom); err != nil { + return nil, fmt.Errorf("parse feed %s: %w", feedURL, err) + } + src := cleanText(atom.Title) + out := make([]FeedItem, 0, len(atom.Entries)) + for _, e := range atom.Entries { + link := atomBestLink(e.Links) + if link == "" { + link = strings.TrimSpace(e.ID) + } + sum := firstNonEmpty(e.Summary, e.Content) + item := FeedItem{ + Title: cleanText(e.Title), + Summary: cleanText(sum), + Link: normalizeURL(link), + Source: src, + } + if item.Title == "" || item.Link == "" { + continue + } + out = append(out, item) + } + if len(out) == 0 { + return nil, fmt.Errorf("empty feed %s", feedURL) + } + return out, nil +} + +func atomBestLink(links []atomLink) string { + var alt, self, any string + for _, l := range links { + href := strings.TrimSpace(l.Href) + if href == "" { + continue + } + any = href + switch strings.ToLower(strings.TrimSpace(l.Rel)) { + case "", "alternate": + if alt == "" { + alt = href + } + case "self": + if self == "" { + self = href + } + } + } + return firstNonEmpty(alt, self, any) +} + +func FormatPostText(item FeedItem) string { + title := strings.TrimSpace(item.Title) + sum := truncateRunes(strings.TrimSpace(item.Summary), summaryRunes) + link := strings.TrimSpace(item.Link) + src := strings.TrimSpace(item.Source) + var b strings.Builder + if src != "" { + b.WriteString(src) + b.WriteString("\n") + } + b.WriteString(title) + if sum != "" && !strings.EqualFold(sum, title) { + b.WriteString("\n\n") + b.WriteString(sum) + } + b.WriteString("\n\n") + b.WriteString(link) + text := strings.TrimSpace(b.String()) + return truncateRunes(text, 1000) +} + +func cleanText(s string) string { + s = html.UnescapeString(s) + s = tagRe.ReplaceAllString(s, " ") + s = strings.Join(strings.Fields(s), " ") + return strings.TrimSpace(s) +} + +func truncateRunes(s string, max int) string { + if max <= 0 || utf8.RuneCountInString(s) <= max { + return s + } + runes := []rune(s) + return string(runes[:max-1]) + "…" +} + +func normalizeURL(raw string) string { + raw = strings.TrimSpace(raw) + if raw == "" { + return "" + } + u, err := url.Parse(raw) + if err != nil || u.Scheme == "" || u.Host == "" { + return "" + } + scheme := strings.ToLower(u.Scheme) + if scheme != "http" && scheme != "https" { + return "" + } + u.Fragment = "" + return u.String() +} + +func firstNonEmpty(vals ...string) string { + for _, v := range vals { + if strings.TrimSpace(v) != "" { + return strings.TrimSpace(v) + } + } + return "" +} diff --git a/internal/module/squarebot/logic/service.go b/internal/module/squarebot/logic/service.go new file mode 100644 index 0000000..a6d1404 --- /dev/null +++ b/internal/module/squarebot/logic/service.go @@ -0,0 +1,213 @@ +package logic + +import ( + "context" + "errors" + "math/rand" + "net/http" + "strings" + "time" + + "go.uber.org/zap" + + squarelogic "spark-server/internal/module/square/logic" + userentity "spark-server/internal/module/user/entity" + "spark-server/internal/pkg/errs" +) + +// Options configures the square media bot worker. +type Options struct { + Enabled bool + Interval time.Duration + TTL time.Duration + Feeds []string + BotIndexes []int + HTTPClient *http.Client +} + +// Users resolves seeded robot accounts. +type Users interface { + GetByEmail(ctx context.Context, email string) (*userentity.User, error) +} + +// Square publishes and expires bot posts. +type Square interface { + PublishBotMedia(ctx context.Context, userID int64, in *squarelogic.BotPublishInput) (*squarelogic.View, error) + ExpireBotPosts(ctx context.Context, now time.Time, limit int) (int, error) +} + +// Locker is an optional distributed tick lock (e.g. Redis SETNX). +type Locker interface { + TryLock(ctx context.Context, key string, ttl time.Duration) (bool, error) +} + +// Service runs one RSS publish + TTL cleanup tick. +type Service struct { + log *zap.Logger + square Square + users Users + locker Locker + opts Options + client *http.Client + rand *rand.Rand + lockKey string +} + +func NewService(log *zap.Logger, square Square, users Users, locker Locker, opts Options) *Service { + if log == nil { + log = zap.NewNop() + } + opts = normalizeOptions(opts) + client := opts.HTTPClient + if client == nil { + client = &http.Client{Timeout: rssTimeout} + } + return &Service{ + log: log, square: square, users: users, locker: locker, opts: opts, + client: client, rand: rand.New(rand.NewSource(time.Now().UnixNano())), + lockKey: "squarebot:tick", + } +} + +func normalizeOptions(o Options) Options { + if o.Interval <= 0 { + o.Interval = 5 * time.Minute + } + if o.TTL <= 0 { + o.TTL = 15 * 24 * time.Hour + } + if len(o.Feeds) == 0 { + o.Feeds = DefaultFeeds() + } + if len(o.BotIndexes) == 0 { + o.BotIndexes = defaultBotIndexes() + } + return o +} + +// DefaultFeeds are public overseas mainstream media RSS endpoints (title/summary/link only). +func DefaultFeeds() []string { + return []string{ + "https://feeds.bbci.co.uk/news/world/rss.xml", + "https://www.theguardian.com/world/rss", + "https://feeds.npr.org/1004/rss.xml", + "https://www.aljazeera.com/xml/rss/all.xml", + "https://rss.nytimes.com/services/xml/rss/nyt/World.xml", + "https://www.reutersagency.com/feed/?best-topics=world-including-us&post_type=best", + } +} + +func defaultBotIndexes() []int { + out := make([]int, 0, userentity.BotCount) + for i := 1; i <= userentity.BotCount; i++ { + out = append(out, i) + } + return out +} + +func (s *Service) Enabled() bool { + return s != nil && s.opts.Enabled && s.square != nil && s.users != nil +} + +func (s *Service) Interval() time.Duration { + if s == nil { + return 5 * time.Minute + } + return s.opts.Interval +} + +// Tick expires old bot posts then publishes one new RSS item via a random bot. +func (s *Service) Tick(ctx context.Context) error { + if !s.Enabled() { + return nil + } + if s.locker != nil { + ok, err := s.locker.TryLock(ctx, s.lockKey, s.opts.Interval) + if err != nil { + s.log.Warn("squarebot lock", zap.Error(err)) + } else if !ok { + return nil + } + } + if n, err := s.square.ExpireBotPosts(ctx, time.Now(), 200); err != nil { + s.log.Warn("squarebot expire", zap.Error(err)) + } else if n > 0 { + s.log.Info("squarebot expired", zap.Int("count", n)) + } + bot, err := s.pickBot(ctx) + if err != nil { + return err + } + item, err := s.pickItem(ctx) + if err != nil { + return err + } + text := FormatPostText(item) + view, err := s.square.PublishBotMedia(ctx, bot.ID, &squarelogic.BotPublishInput{ + Text: text, SourceURL: item.Link, TTL: s.opts.TTL, + }) + if err != nil { + if errors.Is(err, errs.ErrDuplicate) { + s.log.Info("squarebot skip duplicate", zap.String("url", item.Link)) + return nil + } + return err + } + s.log.Info("squarebot posted", + zap.Int64("bot_uid", bot.ID), + zap.String("bot", bot.Nickname), + zap.Int64("post_id", view.ID), + zap.String("url", item.Link), + ) + return nil +} + +func (s *Service) pickBot(ctx context.Context) (*userentity.User, error) { + indexes := append([]int(nil), s.opts.BotIndexes...) + s.rand.Shuffle(len(indexes), func(i, j int) { indexes[i], indexes[j] = indexes[j], indexes[i] }) + var last error + for _, idx := range indexes { + email := userentity.BotEmail(idx) + if email == "" { + continue + } + u, err := s.users.GetByEmail(ctx, email) + if err != nil { + last = err + continue + } + if u != nil && u.ID > 0 { + return u, nil + } + } + if last != nil { + return nil, last + } + return nil, errs.ErrNotFound +} + +func (s *Service) pickItem(ctx context.Context) (FeedItem, error) { + feeds := append([]string(nil), s.opts.Feeds...) + s.rand.Shuffle(len(feeds), func(i, j int) { feeds[i], feeds[j] = feeds[j], feeds[i] }) + var last error + for _, feed := range feeds { + feed = strings.TrimSpace(feed) + if feed == "" { + continue + } + items, err := FetchFeed(ctx, s.client, feed) + if err != nil { + last = err + s.log.Warn("squarebot feed", zap.String("feed", feed), zap.Error(err)) + continue + } + if len(items) == 0 { + continue + } + return items[s.rand.Intn(len(items))], nil + } + if last != nil { + return FeedItem{}, last + } + return FeedItem{}, errs.ErrNotFound +} diff --git a/internal/pkg/errs/codes.go b/internal/pkg/errs/codes.go index 54eab6b..b634abf 100644 --- a/internal/pkg/errs/codes.go +++ b/internal/pkg/errs/codes.go @@ -7,6 +7,7 @@ var ( ErrFrameTooLarge = NewError("40002", "frame too large", "") ErrInvalidFrame = NewError("40003", "invalid frame", "") ErrProxyHeader = NewError("40004", "invalid proxy protocol header", "") + ErrMediaOversized = NewError("40005", "media oversized", "") ErrUnauthorized = NewError("40101", "unauthorized", "") ErrNotAuthenticated = NewError("40102", "not authenticated", "") ErrVerifyCodeInvalid = NewError("40103", "verification code invalid", "") diff --git a/internal/pkg/i18n/catalog_err.go b/internal/pkg/i18n/catalog_err.go index c2d3d86..40375e0 100644 --- a/internal/pkg/i18n/catalog_err.go +++ b/internal/pkg/i18n/catalog_err.go @@ -17,6 +17,9 @@ func init() { "err.40004": { ZhCN: "代理协议错误", En: "invalid proxy protocol header", ZhTW: "代理協定錯誤", }, + "err.40005": { + ZhCN: "媒体过大或分辨率超限", En: "media oversized", ZhTW: "媒體過大或解析度超限", + }, "err.40101": { ZhCN: "未授权", En: "unauthorized", ZhTW: "未授權", }, diff --git a/internal/pkg/i18n/catalog_validate_upload.go b/internal/pkg/i18n/catalog_validate_upload.go index b39b58c..5f67a6b 100644 --- a/internal/pkg/i18n/catalog_validate_upload.go +++ b/internal/pkg/i18n/catalog_validate_upload.go @@ -62,6 +62,11 @@ func init() { KeyUploadDownloadItemsInvalid: { ZhCN: "一次最多签发 9 个下载凭证", En: "At most 9 download credentials per request", ZhTW: "一次最多簽發 9 個下載憑證", }, + KeyMediaOversized: { + ZhCN: "图片最长边须≤1080、视频≤720且体积不超限(头像≤200×200);请先在客户端压缩后再上传", + En: "Image longest edge must be ≤1080, video ≤720 within size limits (avatar ≤200×200); compress on client before upload", + ZhTW: "圖片最長邊須≤1080、影片≤720且體積不超限(頭像≤200×200);請先在客戶端壓縮後再上傳", + }, KeyFavoriteMsgRequired: { ZhCN: "请选择消息", En: "Please choose a message", ZhTW: "請選擇訊息", }, diff --git a/internal/pkg/i18n/keys.go b/internal/pkg/i18n/keys.go index 30b2706..b0c3d50 100644 --- a/internal/pkg/i18n/keys.go +++ b/internal/pkg/i18n/keys.go @@ -102,6 +102,7 @@ const ( KeyUploadItemsRequired = "validate.upload.items_required" KeyUploadItemsInvalid = "validate.upload.items_invalid" KeyUploadDownloadItemsInvalid = "validate.upload.download_items_invalid" + KeyMediaOversized = "err.media_oversized" KeyFavoriteMsgRequired = "validate.favorite.msg_required" KeyFavoriteMsgInvalid = "validate.favorite.msg_invalid" KeyMessageClientMsgRequired = "validate.message.client_msg_required" diff --git a/internal/pkg/media/check.go b/internal/pkg/media/check.go new file mode 100644 index 0000000..bd9699b --- /dev/null +++ b/internal/pkg/media/check.go @@ -0,0 +1,83 @@ +package media + +import "fmt" + +// Kind classifies media for policy checks. +type Kind string + +const ( + KindImage Kind = "image" + KindVideo Kind = "video" + KindAvatar Kind = "avatar" +) + +// Violation describes an oversize resource (dimensions and/or bytes). +type Violation struct { + Kind Kind + Reason string + Width int + Height int + MaxEdge int + Size int64 + MaxSize int64 + Duration int // ms; video only +} + +func (v *Violation) Error() string { + if v == nil { + return "" + } + return fmt.Sprintf("media %s oversize: %s", v.Kind, v.Reason) +} + +// Policy controls user-path detection (bots always compress via infrastructure/media). +type Policy struct { + Enabled bool // master switch for user checks + UserReject bool // when true, publish/upload returns ErrMediaOversized; else log-only + Limits Limits +} + +// CheckImageMeta validates declared image width/height/size against Limits. +// Zero width/height skips dimension check (client omitted); size 0 skips byte check. +func (p Policy) CheckImageMeta(w, h int, size int64) *Violation { + return p.check(KindImage, w, h, size, 0) +} + +// CheckVideoMeta validates declared video metadata. +func (p Policy) CheckVideoMeta(w, h int, size int64, durationMS int) *Violation { + return p.check(KindVideo, w, h, size, durationMS) +} + +// CheckAvatarMeta validates avatar dimensions/size (≤200×200 by default). +func (p Policy) CheckAvatarMeta(w, h int, size int64) *Violation { + return p.check(KindAvatar, w, h, size, 0) +} + +func (p Policy) check(kind Kind, w, h int, size int64, durationMS int) *Violation { + if !p.Enabled { + return nil + } + lim := p.Limits.Normalize() + maxEdge, maxSize := lim.ImageMaxEdge, lim.ImageMaxBytes + switch kind { + case KindVideo: + maxEdge, maxSize = lim.VideoMaxEdge, lim.VideoMaxBytes + case KindAvatar: + maxEdge, maxSize = lim.AvatarMaxEdge, lim.AvatarMaxBytes + } + if w > 0 && h > 0 && NeedsResize(w, h, maxEdge) { + return &Violation{ + Kind: kind, Reason: "edge_exceeded", + Width: w, Height: h, MaxEdge: maxEdge, + Size: size, MaxSize: maxSize, Duration: durationMS, + } + } + if size > 0 && maxSize > 0 && size > maxSize { + return &Violation{ + Kind: kind, Reason: "size_exceeded", + Width: w, Height: h, MaxEdge: maxEdge, + Size: size, MaxSize: maxSize, Duration: durationMS, + } + } + return nil +} diff --git a/internal/pkg/media/check_test.go b/internal/pkg/media/check_test.go new file mode 100644 index 0000000..98f11ec --- /dev/null +++ b/internal/pkg/media/check_test.go @@ -0,0 +1,45 @@ +package media + +import "testing" + +func TestCheckImageMeta(t *testing.T) { + p := Policy{Enabled: true, Limits: DefaultLimits()} + if v := p.CheckImageMeta(1080, 720, 1<<20); v != nil { + t.Fatalf("expected ok, got %+v", v) + } + if v := p.CheckImageMeta(2000, 1000, 1<<20); v == nil || v.Reason != "edge_exceeded" { + t.Fatalf("want edge_exceeded, got %+v", v) + } + if v := p.CheckImageMeta(800, 600, 20<<20); v == nil || v.Reason != "size_exceeded" { + t.Fatalf("want size_exceeded, got %+v", v) + } + p.Enabled = false + if v := p.CheckImageMeta(4000, 4000, 50<<20); v != nil { + t.Fatalf("disabled should skip, got %+v", v) + } +} + +func TestCheckVideoAndAvatar(t *testing.T) { + p := Policy{Enabled: true, Limits: DefaultLimits()} + if v := p.CheckVideoMeta(1280, 720, 10<<20, 5000); v == nil || v.Reason != "edge_exceeded" { + t.Fatalf("1280 edge should fail vs 720: %+v", v) + } + if v := p.CheckVideoMeta(720, 480, 10<<20, 5000); v != nil { + t.Fatalf("720p ok: %+v", v) + } + if v := p.CheckAvatarMeta(200, 200, 100<<10); v != nil { + t.Fatalf("avatar ok: %+v", v) + } + if v := p.CheckAvatarMeta(400, 400, 100<<10); v == nil { + t.Fatal("avatar 400 should fail") + } +} + +func TestNeedsResize(t *testing.T) { + if NeedsResize(1080, 720, 1080) { + t.Fatal("equal edge should not resize") + } + if !NeedsResize(1081, 1, 1080) { + t.Fatal("want resize") + } +} diff --git a/internal/pkg/media/limits.go b/internal/pkg/media/limits.go new file mode 100644 index 0000000..5d438a8 --- /dev/null +++ b/internal/pkg/media/limits.go @@ -0,0 +1,77 @@ +// Package media holds shared media size/resolution policy (no I/O). +// Client App compresses user uploads; bots must compress server-side via infrastructure/media. +package media + +// Default limits aligned with App frontend (image ≤1080p, video ≤720p ≤100MB, avatar ≤200×200). +const ( + DefaultImageMaxEdge = 1080 + DefaultVideoMaxEdge = 720 + DefaultAvatarMaxEdge = 200 + DefaultImageMaxBytes = 10 << 20 // 10 MiB (square/moment image purpose) + DefaultVideoMaxBytes = 100 << 20 // 100 MiB + DefaultAvatarMaxBytes = 2 << 20 // 2 MiB (upload purpose hard cap) + DefaultJPEGQuality = 85 + DefaultVideoCRF = 23 +) + +// Limits is the resolution/size policy shared by user checks and bot compress. +type Limits struct { + ImageMaxEdge int + VideoMaxEdge int + AvatarMaxEdge int + ImageMaxBytes int64 + VideoMaxBytes int64 + AvatarMaxBytes int64 +} + +// DefaultLimits returns client-aligned defaults. +func DefaultLimits() Limits { + return Limits{ + ImageMaxEdge: DefaultImageMaxEdge, + VideoMaxEdge: DefaultVideoMaxEdge, + AvatarMaxEdge: DefaultAvatarMaxEdge, + ImageMaxBytes: DefaultImageMaxBytes, + VideoMaxBytes: DefaultVideoMaxBytes, + AvatarMaxBytes: DefaultAvatarMaxBytes, + } +} + +// Normalize fills zero fields with defaults. +func (l Limits) Normalize() Limits { + d := DefaultLimits() + if l.ImageMaxEdge <= 0 { + l.ImageMaxEdge = d.ImageMaxEdge + } + if l.VideoMaxEdge <= 0 { + l.VideoMaxEdge = d.VideoMaxEdge + } + if l.AvatarMaxEdge <= 0 { + l.AvatarMaxEdge = d.AvatarMaxEdge + } + if l.ImageMaxBytes <= 0 { + l.ImageMaxBytes = d.ImageMaxBytes + } + if l.VideoMaxBytes <= 0 { + l.VideoMaxBytes = d.VideoMaxBytes + } + if l.AvatarMaxBytes <= 0 { + l.AvatarMaxBytes = d.AvatarMaxBytes + } + return l +} + +// LongestEdge returns max(w, h). +func LongestEdge(w, h int) int { + if w > h { + return w + } + return h +} + +// NeedsResize reports whether the frame exceeds maxEdge (already-smaller is kept as-is). +func NeedsResize(w, h, maxEdge int) bool { + if maxEdge <= 0 || w <= 0 || h <= 0 { + return false + } + return LongestEdge(w, h) > maxEdge +} diff --git a/internal/pkg/media/publish.go b/internal/pkg/media/publish.go new file mode 100644 index 0000000..f32f430 --- /dev/null +++ b/internal/pkg/media/publish.go @@ -0,0 +1,34 @@ +package media + +// ImageMeta is client-declared image metadata on publish. +type ImageMeta struct { + Width int + Height int + Size int64 +} + +// VideoMeta is client-declared video metadata on publish. +type VideoMeta struct { + Width int + Height int + Size int64 + DurationMS int +} + +// CheckPublishImages returns the first image violation, if any. +func (p Policy) CheckPublishImages(images []ImageMeta) *Violation { + for _, img := range images { + if v := p.CheckImageMeta(img.Width, img.Height, img.Size); v != nil { + return v + } + } + return nil +} + +// CheckPublishVideo returns a video violation, if any. +func (p Policy) CheckPublishVideo(v *VideoMeta) *Violation { + if v == nil { + return nil + } + return p.CheckVideoMeta(v.Width, v.Height, v.Size, v.DurationMS) +} diff --git a/migrations/0026_square_bot_rss.sql b/migrations/0026_square_bot_rss.sql new file mode 100644 index 0000000..5a9f15f --- /dev/null +++ b/migrations/0026_square_bot_rss.sql @@ -0,0 +1,19 @@ +-- 广场机器人海外媒体 RSS:帖来源、原文 URL、过期时间与 URL 去重。 + +ALTER TABLE t_square_posts + ADD COLUMN source SMALLINT NOT NULL DEFAULT 0 COMMENT '0用户 1机器人海外媒体RSS' AFTER status, + ADD COLUMN source_url VARCHAR(768) NOT NULL DEFAULT '' COMMENT '原文 URL,机器人帖去重键' AFTER source, + ADD COLUMN expires_at DATETIME(3) NULL COMMENT '机器人帖过期时间;到期不可见并清理' AFTER source_url; + +ALTER TABLE t_square_posts + ADD KEY idx_square_posts_expires (expires_at, status), + ADD KEY idx_square_posts_source_url (source_url); + +CREATE TABLE IF NOT EXISTS t_square_bot_links ( + url_hash CHAR(64) NOT NULL COMMENT '规范化 URL 的 SHA-256 hex', + url VARCHAR(768) NOT NULL COMMENT '原文 URL', + post_id BIGINT NOT NULL COMMENT '关联 t_square_posts.id', + created_at DATETIME(3) NOT NULL COMMENT '入库时间', + PRIMARY KEY (url_hash), + KEY idx_square_bot_links_post (post_id) +) COMMENT='广场机器人 RSS 原文 URL 去重';