Compare commits
2 Commits
74a57ed25c
...
f708a86d5c
| Author | SHA1 | Date | |
|---|---|---|---|
| f708a86d5c | |||
| 715ace7ac6 |
74
internal/infrastructure/media/bot.go
Normal file
74
internal/infrastructure/media/bot.go
Normal file
@ -0,0 +1,74 @@
|
||||
package media
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
pkgmedia "spark-server/internal/pkg/media"
|
||||
)
|
||||
|
||||
// BotAsset is compressed media ready for objectstore.Put / PutLarge.
|
||||
type BotAsset struct {
|
||||
Data []byte
|
||||
Width int
|
||||
Height int
|
||||
MIME string
|
||||
Size int64
|
||||
}
|
||||
|
||||
// PrepareBotImage compresses RSS/downloaded image bytes to ≤1080p before object storage write.
|
||||
// Call this after HTTP download and before PutLarge.
|
||||
func PrepareBotImage(src []byte, opt Options) (*BotAsset, error) {
|
||||
res, err := CompressImage(src, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &BotAsset{
|
||||
Data: res.Data, Width: res.Width, Height: res.Height,
|
||||
MIME: res.MIME, Size: int64(len(res.Data)),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// PrepareBotAvatar compresses bot avatar to ≤200×200.
|
||||
func PrepareBotAvatar(src []byte, opt Options) (*BotAsset, error) {
|
||||
res, err := CompressAvatar(src, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &BotAsset{
|
||||
Data: res.Data, Width: res.Width, Height: res.Height,
|
||||
MIME: res.MIME, Size: int64(len(res.Data)),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// PrepareBotVideo compresses downloaded video to ≤720p (requires ffmpeg).
|
||||
func PrepareBotVideo(ctx context.Context, src []byte, opt Options) (*BotAsset, error) {
|
||||
res, err := CompressVideoBytes(ctx, src, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &BotAsset{
|
||||
Data: res.Data, Width: res.Width, Height: res.Height,
|
||||
MIME: res.MIME, Size: int64(len(res.Data)),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// OptionsFromLimits builds compressor options from shared policy limits.
|
||||
func OptionsFromLimits(lim pkgmedia.Limits, jpegQuality, videoCRF int, ffmpegPath string) Options {
|
||||
return Options{
|
||||
Limits: lim.Normalize(),
|
||||
JPEGQuality: jpegQuality,
|
||||
VideoCRF: videoCRF,
|
||||
FFmpegPath: strings.TrimSpace(ffmpegPath),
|
||||
}
|
||||
}
|
||||
|
||||
// MustCompressBotImage is PrepareBotImage with a clearer error prefix for callers.
|
||||
func MustCompressBotImage(src []byte, opt Options) (*BotAsset, error) {
|
||||
a, err := PrepareBotImage(src, opt)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("bot image compress: %w", err)
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
124
internal/infrastructure/media/image.go
Normal file
124
internal/infrastructure/media/image.go
Normal file
@ -0,0 +1,124 @@
|
||||
// Package media compresses images (stdlib + x/image) and videos (ffmpeg) for bot pipelines.
|
||||
// Deploy note: video compression requires the `ffmpeg` binary on PATH (or Options.FFmpegPath).
|
||||
package media
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"image"
|
||||
_ "image/gif"
|
||||
"image/jpeg"
|
||||
_ "image/png"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/image/draw"
|
||||
|
||||
pkgmedia "spark-server/internal/pkg/media"
|
||||
)
|
||||
|
||||
// Options configures compressors.
|
||||
type Options struct {
|
||||
Limits pkgmedia.Limits
|
||||
JPEGQuality int
|
||||
VideoCRF int
|
||||
FFmpegPath string // default "ffmpeg"
|
||||
}
|
||||
|
||||
func (o Options) normalize() Options {
|
||||
o.Limits = o.Limits.Normalize()
|
||||
if o.JPEGQuality <= 0 || o.JPEGQuality > 100 {
|
||||
o.JPEGQuality = pkgmedia.DefaultJPEGQuality
|
||||
}
|
||||
if o.VideoCRF <= 0 {
|
||||
o.VideoCRF = pkgmedia.DefaultVideoCRF
|
||||
}
|
||||
if strings.TrimSpace(o.FFmpegPath) == "" {
|
||||
o.FFmpegPath = "ffmpeg"
|
||||
}
|
||||
return o
|
||||
}
|
||||
|
||||
// ImageResult is compressed image output.
|
||||
type ImageResult struct {
|
||||
Data []byte
|
||||
Width int
|
||||
Height int
|
||||
MIME string
|
||||
}
|
||||
|
||||
// CompressImage scales longest edge to ≤ Limits.ImageMaxEdge and re-encodes as JPEG (PNG in → JPEG out unless already small PNG kept as PNG when no resize needed and source is PNG).
|
||||
func CompressImage(src []byte, opt Options) (*ImageResult, error) {
|
||||
opt = opt.normalize()
|
||||
if len(src) == 0 {
|
||||
return nil, fmt.Errorf("media: empty image")
|
||||
}
|
||||
img, format, err := image.Decode(bytes.NewReader(src))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("media: decode image: %w", err)
|
||||
}
|
||||
b := img.Bounds()
|
||||
w, h := b.Dx(), b.Dy()
|
||||
maxEdge := opt.Limits.ImageMaxEdge
|
||||
if !pkgmedia.NeedsResize(w, h, maxEdge) && int64(len(src)) <= opt.Limits.ImageMaxBytes {
|
||||
mime := mimeForFormat(format)
|
||||
return &ImageResult{Data: src, Width: w, Height: h, MIME: mime}, nil
|
||||
}
|
||||
nw, nh := scaleToMax(w, h, maxEdge)
|
||||
dst := image.NewRGBA(image.Rect(0, 0, nw, nh))
|
||||
draw.CatmullRom.Scale(dst, dst.Bounds(), img, b, draw.Over, nil)
|
||||
var buf bytes.Buffer
|
||||
if err := jpeg.Encode(&buf, dst, &jpeg.Options{Quality: opt.JPEGQuality}); err != nil {
|
||||
return nil, fmt.Errorf("media: encode jpeg: %w", err)
|
||||
}
|
||||
out := buf.Bytes()
|
||||
if int64(len(out)) > opt.Limits.ImageMaxBytes {
|
||||
return nil, fmt.Errorf("media: compressed image still too large (%d > %d)", len(out), opt.Limits.ImageMaxBytes)
|
||||
}
|
||||
return &ImageResult{Data: out, Width: nw, Height: nh, MIME: "image/jpeg"}, nil
|
||||
}
|
||||
|
||||
// CompressAvatar scales to ≤ AvatarMaxEdge and JPEG-encodes.
|
||||
func CompressAvatar(src []byte, opt Options) (*ImageResult, error) {
|
||||
opt = opt.normalize()
|
||||
lim := opt.Limits
|
||||
lim.ImageMaxEdge = lim.AvatarMaxEdge
|
||||
lim.ImageMaxBytes = lim.AvatarMaxBytes
|
||||
opt.Limits = lim
|
||||
return CompressImage(src, opt)
|
||||
}
|
||||
|
||||
func scaleToMax(w, h, maxEdge int) (int, int) {
|
||||
if maxEdge <= 0 || !pkgmedia.NeedsResize(w, h, maxEdge) {
|
||||
return w, h
|
||||
}
|
||||
edge := pkgmedia.LongestEdge(w, h)
|
||||
nw := w * maxEdge / edge
|
||||
nh := h * maxEdge / edge
|
||||
if nw < 1 {
|
||||
nw = 1
|
||||
}
|
||||
if nh < 1 {
|
||||
nh = 1
|
||||
}
|
||||
return nw, nh
|
||||
}
|
||||
|
||||
func mimeForFormat(format string) string {
|
||||
switch strings.ToLower(format) {
|
||||
case "png":
|
||||
return "image/png"
|
||||
case "gif":
|
||||
return "image/gif"
|
||||
default:
|
||||
return "image/jpeg"
|
||||
}
|
||||
}
|
||||
|
||||
// EncodePNG is a test helper / bot helper for synthetic images.
|
||||
func EncodePNG(img image.Image) ([]byte, error) {
|
||||
var buf bytes.Buffer
|
||||
if err := png.Encode(&buf, img); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
71
internal/infrastructure/media/image_test.go
Normal file
71
internal/infrastructure/media/image_test.go
Normal file
@ -0,0 +1,71 @@
|
||||
package media
|
||||
|
||||
import (
|
||||
"image"
|
||||
"image/color"
|
||||
"testing"
|
||||
|
||||
pkgmedia "spark-server/internal/pkg/media"
|
||||
)
|
||||
|
||||
func TestCompressImageDownscales(t *testing.T) {
|
||||
img := image.NewRGBA(image.Rect(0, 0, 2000, 1000))
|
||||
for y := 0; y < 1000; y++ {
|
||||
for x := 0; x < 2000; x++ {
|
||||
img.Set(x, y, color.RGBA{R: uint8(x % 255), G: uint8(y % 255), B: 40, A: 255})
|
||||
}
|
||||
}
|
||||
src, err := EncodePNG(img)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
res, err := CompressImage(src, Options{Limits: pkgmedia.DefaultLimits(), JPEGQuality: 80})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if pkgmedia.LongestEdge(res.Width, res.Height) > pkgmedia.DefaultImageMaxEdge {
|
||||
t.Fatalf("edge %dx%d still over 1080", res.Width, res.Height)
|
||||
}
|
||||
if res.MIME != "image/jpeg" {
|
||||
t.Fatalf("mime %s", res.MIME)
|
||||
}
|
||||
small := image.NewRGBA(image.Rect(0, 0, 100, 80))
|
||||
smallSrc, err := EncodePNG(small)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
keep, err := CompressImage(smallSrc, Options{Limits: pkgmedia.DefaultLimits()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if keep.Width != 100 || keep.Height != 80 {
|
||||
t.Fatalf("should keep small image dims, got %dx%d", keep.Width, keep.Height)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompressAvatar(t *testing.T) {
|
||||
img := image.NewRGBA(image.Rect(0, 0, 500, 500))
|
||||
src, err := EncodePNG(img)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
res, err := CompressAvatar(src, Options{Limits: pkgmedia.DefaultLimits()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if pkgmedia.LongestEdge(res.Width, res.Height) > pkgmedia.DefaultAvatarMaxEdge {
|
||||
t.Fatalf("avatar %dx%d", res.Width, res.Height)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareBotImage(t *testing.T) {
|
||||
img := image.NewRGBA(image.Rect(0, 0, 1600, 900))
|
||||
src, err := EncodePNG(img)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a, err := PrepareBotImage(src, Options{Limits: pkgmedia.DefaultLimits()})
|
||||
if err != nil || a == nil || a.Size <= 0 {
|
||||
t.Fatalf("%+v %v", a, err)
|
||||
}
|
||||
}
|
||||
148
internal/infrastructure/media/video.go
Normal file
148
internal/infrastructure/media/video.go
Normal file
@ -0,0 +1,148 @@
|
||||
package media
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
pkgmedia "spark-server/internal/pkg/media"
|
||||
)
|
||||
|
||||
// VideoResult is compressed video output (H.264 MP4).
|
||||
type VideoResult struct {
|
||||
Data []byte
|
||||
Width int
|
||||
Height int
|
||||
MIME string
|
||||
}
|
||||
|
||||
// CompressVideoBytes writes src to a temp file, runs ffmpeg scale+libx264, returns MP4 bytes.
|
||||
// Requires ffmpeg on PATH (or Options.FFmpegPath). Returns error if ffmpeg is missing.
|
||||
func CompressVideoBytes(ctx context.Context, src []byte, opt Options) (*VideoResult, error) {
|
||||
opt = opt.normalize()
|
||||
if len(src) == 0 {
|
||||
return nil, fmt.Errorf("media: empty video")
|
||||
}
|
||||
if int64(len(src)) > opt.Limits.VideoMaxBytes*2 {
|
||||
// refuse absurd remote downloads before temp write
|
||||
return nil, fmt.Errorf("media: video source too large")
|
||||
}
|
||||
dir, err := os.MkdirTemp("", "spark-media-*")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = os.RemoveAll(dir) }()
|
||||
inPath := filepath.Join(dir, "in.bin")
|
||||
outPath := filepath.Join(dir, "out.mp4")
|
||||
if err := os.WriteFile(inPath, src, 0o600); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
w, h, err := probeVideoSize(ctx, opt.FFmpegPath, inPath)
|
||||
if err != nil {
|
||||
// ffprobe may be absent; fall through to always re-encode
|
||||
w, h = 0, 0
|
||||
}
|
||||
if w > 0 && h > 0 && !pkgmedia.NeedsResize(w, h, opt.Limits.VideoMaxEdge) && int64(len(src)) <= opt.Limits.VideoMaxBytes {
|
||||
return &VideoResult{Data: src, Width: w, Height: h, MIME: "video/mp4"}, nil
|
||||
}
|
||||
if err := runFFmpegCompress(ctx, opt, inPath, outPath); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out, err := os.ReadFile(outPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if int64(len(out)) > opt.Limits.VideoMaxBytes {
|
||||
return nil, fmt.Errorf("media: compressed video still too large (%d > %d)", len(out), opt.Limits.VideoMaxBytes)
|
||||
}
|
||||
ow, oh, _ := probeVideoSize(ctx, opt.FFmpegPath, outPath)
|
||||
if ow <= 0 {
|
||||
ow, oh = w, h
|
||||
}
|
||||
return &VideoResult{Data: out, Width: ow, Height: oh, MIME: "video/mp4"}, nil
|
||||
}
|
||||
|
||||
func runFFmpegCompress(ctx context.Context, opt Options, inPath, outPath string) error {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if _, ok := ctx.Deadline(); !ok {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithTimeout(ctx, 3*time.Minute)
|
||||
defer cancel()
|
||||
}
|
||||
maxEdge := opt.Limits.VideoMaxEdge
|
||||
// Longest side ≤ maxEdge; force_original_aspect_ratio keeps aspect, even dims for H.264.
|
||||
vf := fmt.Sprintf("scale=%d:%d:force_original_aspect_ratio=decrease,pad=ceil(iw/2)*2:ceil(ih/2)*2", maxEdge, maxEdge)
|
||||
args := []string{
|
||||
"-y", "-hide_banner", "-loglevel", "error",
|
||||
"-i", inPath,
|
||||
"-vf", vf,
|
||||
"-c:v", "libx264", "-crf", strconv.Itoa(opt.VideoCRF), "-preset", "veryfast",
|
||||
"-c:a", "aac", "-b:a", "128k",
|
||||
"-movflags", "+faststart",
|
||||
outPath,
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, opt.FFmpegPath, args...)
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stderr = &stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
msg := strings.TrimSpace(stderr.String())
|
||||
if msg == "" {
|
||||
msg = err.Error()
|
||||
}
|
||||
if isExecNotFound(err) {
|
||||
return fmt.Errorf("media: ffmpeg not found (install ffmpeg for bot video compress): %w", err)
|
||||
}
|
||||
return fmt.Errorf("media: ffmpeg: %s", msg)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func probeVideoSize(ctx context.Context, ffmpegPath, path string) (w, h int, err error) {
|
||||
// Prefer ffprobe beside ffmpeg; fall back to ffmpeg -i parse.
|
||||
probe := strings.Replace(ffmpegPath, "ffmpeg", "ffprobe", 1)
|
||||
if probe == ffmpegPath {
|
||||
probe = "ffprobe"
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, probe,
|
||||
"-v", "error", "-select_streams", "v:0",
|
||||
"-show_entries", "stream=width,height", "-of", "csv=p=0:s=x", path,
|
||||
)
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
parts := strings.Split(strings.TrimSpace(string(out)), "x")
|
||||
if len(parts) != 2 {
|
||||
return 0, 0, fmt.Errorf("media: bad ffprobe output")
|
||||
}
|
||||
w, _ = strconv.Atoi(parts[0])
|
||||
h, _ = strconv.Atoi(parts[1])
|
||||
if w <= 0 || h <= 0 {
|
||||
return 0, 0, fmt.Errorf("media: invalid video size")
|
||||
}
|
||||
return w, h, nil
|
||||
}
|
||||
|
||||
func isExecNotFound(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
return strings.Contains(err.Error(), "executable file not found") ||
|
||||
strings.Contains(err.Error(), "no such file") ||
|
||||
os.IsNotExist(err)
|
||||
}
|
||||
|
||||
// FFmpegAvailable reports whether the configured ffmpeg binary can be executed.
|
||||
func FFmpegAvailable(opt Options) bool {
|
||||
opt = opt.normalize()
|
||||
path, err := exec.LookPath(opt.FFmpegPath)
|
||||
return err == nil && path != ""
|
||||
}
|
||||
@ -21,10 +21,10 @@ import (
|
||||
const (
|
||||
maxGetBytes = 32 << 20
|
||||
maxPutBytes = 4 << 20
|
||||
maxPutLarge = 50 << 20 // admin music audio etc.
|
||||
maxPutLarge = 100 << 20 // admin music / bot video after compress (≤100 MiB)
|
||||
)
|
||||
|
||||
// PutLarge writes an object up to maxBytes (capped at 50 MiB).
|
||||
// PutLarge writes an object up to maxBytes (capped at 100 MiB).
|
||||
func (s *Store) PutLarge(ctx context.Context, key string, body []byte, contentType string, maxBytes int) error {
|
||||
if !s.Ready() {
|
||||
return errs.ErrAvailable
|
||||
|
||||
@ -38,6 +38,7 @@ func envelopeCode(lang i18n.Lang) []entity.Item {
|
||||
code(lang, "frame_too_large", errs.ErrFrameTooLarge),
|
||||
code(lang, "invalid_frame", errs.ErrInvalidFrame),
|
||||
code(lang, "invalid_proxy_header", errs.ErrProxyHeader),
|
||||
code(lang, "media_oversized", errs.ErrMediaOversized),
|
||||
code(lang, "unauthorized", errs.ErrUnauthorized),
|
||||
code(lang, "not_authenticated", errs.ErrNotAuthenticated),
|
||||
code(lang, "verify_code_invalid", errs.ErrVerifyCodeInvalid),
|
||||
|
||||
55
internal/module/moment/logic/media_check.go
Normal file
55
internal/module/moment/logic/media_check.go
Normal file
@ -0,0 +1,55 @@
|
||||
package logic
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"spark-server/internal/module/moment/entity"
|
||||
"spark-server/internal/pkg/errs"
|
||||
"spark-server/internal/pkg/i18n"
|
||||
"spark-server/internal/pkg/media"
|
||||
"spark-server/internal/pkg/reqctx"
|
||||
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// checkMediaMeta validates client-declared w/h/size on user publish.
|
||||
func (s *Service) checkMediaMeta(ctx context.Context, images []*entity.Image, video *entity.Video) error {
|
||||
if s == nil || !s.media.Enabled {
|
||||
return nil
|
||||
}
|
||||
var v *media.Violation
|
||||
if video != nil {
|
||||
v = s.media.CheckPublishVideo(&media.VideoMeta{
|
||||
Width: video.Width, Height: video.Height, Size: video.Size, DurationMS: video.DurationMS,
|
||||
})
|
||||
} else {
|
||||
metas := make([]media.ImageMeta, 0, len(images))
|
||||
for _, img := range images {
|
||||
if img == nil {
|
||||
continue
|
||||
}
|
||||
metas = append(metas, media.ImageMeta{Width: img.Width, Height: img.Height, Size: img.Size})
|
||||
}
|
||||
v = s.media.CheckPublishImages(metas)
|
||||
}
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
log := s.log
|
||||
if log == nil {
|
||||
log = zap.NewNop()
|
||||
}
|
||||
log.Warn("moment media oversize",
|
||||
zap.String("kind", string(v.Kind)),
|
||||
zap.String("reason", v.Reason),
|
||||
zap.Int("w", v.Width),
|
||||
zap.Int("h", v.Height),
|
||||
zap.Int("max_edge", v.MaxEdge),
|
||||
zap.Int64("size", v.Size),
|
||||
zap.Int64("max_size", v.MaxSize),
|
||||
)
|
||||
if !s.media.UserReject {
|
||||
return nil
|
||||
}
|
||||
return errs.ErrMediaOversized.WithMsg(i18n.T(reqctx.Lang(ctx), i18n.KeyMediaOversized, nil))
|
||||
}
|
||||
@ -40,6 +40,9 @@ func (s *Service) Publish(ctx context.Context, userID int64, in *PublishInput) (
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.checkMediaMeta(ctx, in.Images, in.Video); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, err := content.Marshal()
|
||||
if err != nil {
|
||||
return nil, errs.ErrInvalidArgument.WithMsg(i18n.T(reqctx.Lang(ctx), i18n.KeyMomentContentInvalid, nil))
|
||||
|
||||
@ -6,6 +6,9 @@ import (
|
||||
friendentity "spark-server/internal/module/friend/entity"
|
||||
"spark-server/internal/module/moment/entity"
|
||||
userentity "spark-server/internal/module/user/entity"
|
||||
"spark-server/internal/pkg/media"
|
||||
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// Repository persists Moments. Model implements it.
|
||||
@ -71,10 +74,23 @@ type Service struct {
|
||||
objects Objects
|
||||
ids IDGen
|
||||
burst *burst
|
||||
media media.Policy
|
||||
log *zap.Logger
|
||||
}
|
||||
|
||||
func NewService(repo Repository, friends Friends, users Users, objects Objects, ids IDGen) *Service {
|
||||
return &Service{repo: repo, friends: friends, users: users, objects: objects, ids: ids, burst: newBurst()}
|
||||
return &Service{repo: repo, friends: friends, users: users, objects: objects, ids: ids, burst: newBurst(), log: zap.NewNop()}
|
||||
}
|
||||
|
||||
// ConfigureMedia sets user-path media detection (client compresses; server checks declared meta).
|
||||
func (s *Service) ConfigureMedia(p media.Policy, log *zap.Logger) {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.media = p
|
||||
if log != nil {
|
||||
s.log = log
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) ready() bool {
|
||||
|
||||
@ -11,11 +11,25 @@ type Post struct {
|
||||
Visibility Visibility
|
||||
AuditStatus AuditStatus
|
||||
Status Status
|
||||
Source Source
|
||||
SourceURL string
|
||||
ExpiresAtMS int64 // 0 = never expires
|
||||
LikeCount int
|
||||
CommentCount int
|
||||
CreatedAtMS int64
|
||||
}
|
||||
|
||||
// Expired reports whether a bot post has passed expires_at (ms).
|
||||
func (p *Post) Expired(nowMS int64) bool {
|
||||
if p == nil || p.ExpiresAtMS <= 0 {
|
||||
return false
|
||||
}
|
||||
if nowMS <= 0 {
|
||||
return false
|
||||
}
|
||||
return nowMS >= p.ExpiresAtMS
|
||||
}
|
||||
|
||||
// POI is an optional location pin.
|
||||
type POI struct {
|
||||
Lat float64
|
||||
|
||||
18
internal/module/square/entity/source.go
Normal file
18
internal/module/square/entity/source.go
Normal file
@ -0,0 +1,18 @@
|
||||
package entity
|
||||
|
||||
// Source is square_posts.source.
|
||||
type Source int16
|
||||
|
||||
const (
|
||||
SourceUser Source = 0
|
||||
SourceBotRSS Source = 1
|
||||
)
|
||||
|
||||
func (s Source) Valid() bool {
|
||||
return s == SourceUser || s == SourceBotRSS
|
||||
}
|
||||
|
||||
// IsBot reports whether the post was published by the media bot worker.
|
||||
func (s Source) IsBot() bool {
|
||||
return s == SourceBotRSS
|
||||
}
|
||||
83
internal/module/square/logic/bot_publish.go
Normal file
83
internal/module/square/logic/bot_publish.go
Normal file
@ -0,0 +1,83 @@
|
||||
package logic
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"spark-server/internal/module/square/entity"
|
||||
"spark-server/internal/pkg/errs"
|
||||
"spark-server/internal/pkg/i18n"
|
||||
"spark-server/internal/pkg/reqctx"
|
||||
)
|
||||
|
||||
// BotPublishInput is an internal publish from the media bot worker (no rate limit).
|
||||
type BotPublishInput struct {
|
||||
Text string
|
||||
SourceURL string
|
||||
TTL time.Duration
|
||||
}
|
||||
|
||||
// PublishBotMedia publishes a public text post as a bot user with TTL and URL dedupe.
|
||||
func (s *Service) PublishBotMedia(ctx context.Context, userID int64, in *BotPublishInput) (*View, error) {
|
||||
if !s.ready() || userID <= 0 {
|
||||
return nil, errs.ErrUnauthorized
|
||||
}
|
||||
if in == nil {
|
||||
return nil, errs.ErrInvalidArgument
|
||||
}
|
||||
text := strings.TrimSpace(in.Text)
|
||||
url := strings.TrimSpace(in.SourceURL)
|
||||
if text == "" || url == "" {
|
||||
return nil, errs.ErrInvalidArgument.WithMsg(i18n.T(reqctx.Lang(ctx), i18n.KeySquareEmpty, nil))
|
||||
}
|
||||
if utf8.RuneCountInString(text) > entity.MaxTextRunes {
|
||||
return nil, errs.ErrInvalidArgument.WithMsg(i18n.T(reqctx.Lang(ctx), i18n.KeySquareTextInvalid, nil))
|
||||
}
|
||||
ttl := in.TTL
|
||||
if ttl <= 0 {
|
||||
ttl = 15 * 24 * time.Hour
|
||||
}
|
||||
content := &entity.Content{}
|
||||
raw, err := content.Marshal()
|
||||
if err != nil {
|
||||
return nil, errs.ErrInvalidArgument.WithMsg(i18n.T(reqctx.Lang(ctx), i18n.KeySquareContentInvalid, nil))
|
||||
}
|
||||
id, err := s.ids.NextID()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
now := time.Now()
|
||||
post := &entity.Post{
|
||||
ID: id, UserID: userID, Kind: entity.KindText, Text: text, Content: raw,
|
||||
Visibility: entity.VisibilityPublic, AuditStatus: entity.AuditApproved,
|
||||
Status: entity.StatusNormal, Source: entity.SourceBotRSS, SourceURL: url,
|
||||
ExpiresAtMS: now.Add(ttl).UnixMilli(), CreatedAtMS: now.UnixMilli(),
|
||||
}
|
||||
if err := s.repo.CreateBotPost(ctx, post); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.viewOf(ctx, userID, post, true)
|
||||
}
|
||||
|
||||
// ExpireBotPosts soft-deletes expired bot posts and best-effort deletes media.
|
||||
func (s *Service) ExpireBotPosts(ctx context.Context, now time.Time, limit int) (int, error) {
|
||||
if !s.ready() {
|
||||
return 0, nil
|
||||
}
|
||||
if now.IsZero() {
|
||||
now = time.Now()
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = 200
|
||||
}
|
||||
posts, err := s.repo.SoftDeleteExpiredBotPosts(ctx, now.UnixMilli(), limit)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
for _, post := range posts {
|
||||
s.deleteMedia(ctx, post)
|
||||
}
|
||||
return len(posts), nil
|
||||
}
|
||||
56
internal/module/square/logic/media_check.go
Normal file
56
internal/module/square/logic/media_check.go
Normal file
@ -0,0 +1,56 @@
|
||||
package logic
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"spark-server/internal/module/square/entity"
|
||||
"spark-server/internal/pkg/errs"
|
||||
"spark-server/internal/pkg/i18n"
|
||||
"spark-server/internal/pkg/media"
|
||||
"spark-server/internal/pkg/reqctx"
|
||||
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// checkMediaMeta validates client-declared w/h/size on user publish.
|
||||
// Default: log only; media.user_reject=true returns ErrMediaOversized (40005).
|
||||
func (s *Service) checkMediaMeta(ctx context.Context, images []*entity.Image, video *entity.Video) error {
|
||||
if s == nil || !s.media.Enabled {
|
||||
return nil
|
||||
}
|
||||
var v *media.Violation
|
||||
if video != nil {
|
||||
v = s.media.CheckPublishVideo(&media.VideoMeta{
|
||||
Width: video.Width, Height: video.Height, Size: video.Size, DurationMS: video.DurationMS,
|
||||
})
|
||||
} else {
|
||||
metas := make([]media.ImageMeta, 0, len(images))
|
||||
for _, img := range images {
|
||||
if img == nil {
|
||||
continue
|
||||
}
|
||||
metas = append(metas, media.ImageMeta{Width: img.Width, Height: img.Height, Size: img.Size})
|
||||
}
|
||||
v = s.media.CheckPublishImages(metas)
|
||||
}
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
log := s.log
|
||||
if log == nil {
|
||||
log = zap.NewNop()
|
||||
}
|
||||
log.Warn("square media oversize",
|
||||
zap.String("kind", string(v.Kind)),
|
||||
zap.String("reason", v.Reason),
|
||||
zap.Int("w", v.Width),
|
||||
zap.Int("h", v.Height),
|
||||
zap.Int("max_edge", v.MaxEdge),
|
||||
zap.Int64("size", v.Size),
|
||||
zap.Int64("max_size", v.MaxSize),
|
||||
)
|
||||
if !s.media.UserReject {
|
||||
return nil
|
||||
}
|
||||
return errs.ErrMediaOversized.WithMsg(i18n.T(reqctx.Lang(ctx), i18n.KeyMediaOversized, nil))
|
||||
}
|
||||
@ -40,6 +40,9 @@ func (s *Service) Publish(ctx context.Context, userID int64, in *PublishInput) (
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.checkMediaMeta(ctx, in.Images, in.Video); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, err := content.Marshal()
|
||||
if err != nil {
|
||||
return nil, errs.ErrInvalidArgument.WithMsg(i18n.T(reqctx.Lang(ctx), i18n.KeySquareContentInvalid, nil))
|
||||
|
||||
@ -6,6 +6,9 @@ import (
|
||||
friendentity "spark-server/internal/module/friend/entity"
|
||||
"spark-server/internal/module/square/entity"
|
||||
userentity "spark-server/internal/module/user/entity"
|
||||
"spark-server/internal/pkg/media"
|
||||
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// Repository persists square data. Model implements it.
|
||||
@ -64,10 +67,23 @@ type Service struct {
|
||||
objects Objects
|
||||
ids IDGen
|
||||
burst *burst
|
||||
media media.Policy
|
||||
log *zap.Logger
|
||||
}
|
||||
|
||||
func NewService(repo Repository, friends Friends, users Users, objects Objects, ids IDGen) *Service {
|
||||
return &Service{repo: repo, friends: friends, users: users, objects: objects, ids: ids, burst: newBurst()}
|
||||
return &Service{repo: repo, friends: friends, users: users, objects: objects, ids: ids, burst: newBurst(), log: zap.NewNop()}
|
||||
}
|
||||
|
||||
// ConfigureMedia sets user-path media detection (bots compress via squarebot MediaCompressor).
|
||||
func (s *Service) ConfigureMedia(p media.Policy, log *zap.Logger) {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.media = p
|
||||
if log != nil {
|
||||
s.log = log
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) ready() bool {
|
||||
|
||||
@ -2,6 +2,7 @@ package logic
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"spark-server/internal/module/square/entity"
|
||||
)
|
||||
@ -16,6 +17,9 @@ func (v *visCtx) canSee(p *entity.Post) bool {
|
||||
if p == nil || p.Status != entity.StatusNormal || p.AuditStatus != entity.AuditApproved {
|
||||
return false
|
||||
}
|
||||
if p.Expired(time.Now().UnixMilli()) {
|
||||
return false
|
||||
}
|
||||
if p.UserID != v.viewer && v.blocked[p.UserID] {
|
||||
return false
|
||||
}
|
||||
|
||||
103
internal/module/square/model/bot.go
Normal file
103
internal/module/square/model/bot.go
Normal file
@ -0,0 +1,103 @@
|
||||
package model
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
|
||||
"spark-server/internal/module/square/entity"
|
||||
"spark-server/internal/pkg/errs"
|
||||
)
|
||||
|
||||
// CreateBotPost inserts a bot RSS post and claims its source URL atomically.
|
||||
func (m *Repo) CreateBotPost(ctx context.Context, post *entity.Post) error {
|
||||
if post == nil || post.ID <= 0 || post.UserID <= 0 {
|
||||
return errs.ErrInvalidArgument
|
||||
}
|
||||
url := strings.TrimSpace(post.SourceURL)
|
||||
if url == "" {
|
||||
return errs.ErrInvalidArgument
|
||||
}
|
||||
post.SourceURL = url
|
||||
if post.Source == 0 {
|
||||
post.Source = entity.SourceBotRSS
|
||||
}
|
||||
db, err := m.session(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
row := postFrom(post)
|
||||
if row.CreatedAt.IsZero() {
|
||||
row.CreatedAt = time.Now()
|
||||
}
|
||||
hash := urlHash(url)
|
||||
return db.Transaction(func(tx *gorm.DB) error {
|
||||
link := BotLink{URLHash: hash, URL: url, PostID: post.ID, CreatedAt: row.CreatedAt}
|
||||
res := tx.Clauses(clause.OnConflict{DoNothing: true}).Create(&link)
|
||||
if res.Error != nil {
|
||||
return mapDBErr(res.Error)
|
||||
}
|
||||
if res.RowsAffected == 0 {
|
||||
return errs.ErrDuplicate
|
||||
}
|
||||
if err := tx.Create(row).Error; err != nil {
|
||||
return mapDBErr(err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// SoftDeleteExpiredBotPosts marks expired bot posts deleted; returns deleted rows for media GC.
|
||||
func (m *Repo) SoftDeleteExpiredBotPosts(ctx context.Context, nowMS int64, limit int) ([]*entity.Post, error) {
|
||||
if limit <= 0 {
|
||||
limit = 100
|
||||
}
|
||||
if nowMS <= 0 {
|
||||
nowMS = time.Now().UnixMilli()
|
||||
}
|
||||
db, err := m.session(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
deadline := time.UnixMilli(nowMS)
|
||||
var rows []Post
|
||||
err = db.Transaction(func(tx *gorm.DB) error {
|
||||
q := tx.Where(
|
||||
"source = ? AND status = ? AND expires_at IS NOT NULL AND expires_at <= ?",
|
||||
entity.SourceBotRSS, entity.StatusNormal, deadline,
|
||||
).Order("expires_at ASC").Limit(limit)
|
||||
if err := q.Find(&rows).Error; err != nil {
|
||||
return mapDBErr(err)
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
ids := make([]int64, 0, len(rows))
|
||||
for i := range rows {
|
||||
ids = append(ids, rows[i].ID)
|
||||
}
|
||||
res := tx.Model(&Post{}).Where("id IN ? AND status = ?", ids, entity.StatusNormal).
|
||||
Update("status", entity.StatusDeleted)
|
||||
if res.Error != nil {
|
||||
return mapDBErr(res.Error)
|
||||
}
|
||||
for i := range rows {
|
||||
rows[i].Status = int16(entity.StatusDeleted)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return collectPosts(rows), nil
|
||||
}
|
||||
|
||||
func urlHash(url string) string {
|
||||
sum := sha256.Sum256([]byte(strings.TrimSpace(url)))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"spark-server/internal/module/square/entity"
|
||||
"spark-server/internal/pkg/errs"
|
||||
@ -23,6 +24,7 @@ type MemoryRepo struct {
|
||||
comments map[int64]*entity.Comment
|
||||
notices map[int64]*entity.Notice
|
||||
noticeState map[int64]*entity.NoticeState
|
||||
botLinks map[string]int64 // url -> post_id
|
||||
}
|
||||
|
||||
func NewMemoryRepo() *MemoryRepo {
|
||||
@ -33,6 +35,7 @@ func NewMemoryRepo() *MemoryRepo {
|
||||
comments: make(map[int64]*entity.Comment),
|
||||
notices: make(map[int64]*entity.Notice),
|
||||
noticeState: make(map[int64]*entity.NoticeState),
|
||||
botLinks: make(map[string]int64),
|
||||
}
|
||||
}
|
||||
|
||||
@ -50,11 +53,15 @@ func (m *MemoryRepo) ListByAuthors(_ context.Context, authorIDs []int64, cursor
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
allow := toSet(authorIDs)
|
||||
nowMS := time.Now().UnixMilli()
|
||||
out := make([]*entity.Post, 0)
|
||||
for _, row := range m.posts {
|
||||
if row.Status != entity.StatusNormal || !allow[row.UserID] {
|
||||
continue
|
||||
}
|
||||
if row.Expired(nowMS) {
|
||||
continue
|
||||
}
|
||||
if cursor > 0 && row.ID >= cursor {
|
||||
continue
|
||||
}
|
||||
@ -133,6 +140,7 @@ func (m *MemoryRepo) GetComment(_ context.Context, id int64) (*entity.Comment, e
|
||||
func (m *MemoryRepo) ListPublicFeed(_ context.Context, cursor int64, limit int) ([]*entity.Post, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
nowMS := time.Now().UnixMilli()
|
||||
out := make([]*entity.Post, 0)
|
||||
for _, row := range m.posts {
|
||||
if row.Status != entity.StatusNormal || row.AuditStatus != entity.AuditApproved {
|
||||
@ -141,6 +149,9 @@ func (m *MemoryRepo) ListPublicFeed(_ context.Context, cursor int64, limit int)
|
||||
if row.Visibility != entity.VisibilityPublic && row.Visibility != entity.VisibilityExclude {
|
||||
continue
|
||||
}
|
||||
if row.Expired(nowMS) {
|
||||
continue
|
||||
}
|
||||
if cursor > 0 && row.ID >= cursor {
|
||||
continue
|
||||
}
|
||||
|
||||
@ -2,6 +2,7 @@ package model
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"spark-server/internal/module/square/entity"
|
||||
@ -24,6 +25,57 @@ func (m *MemoryRepo) CreatePost(_ context.Context, post *entity.Post, audience [
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MemoryRepo) CreateBotPost(_ context.Context, post *entity.Post) error {
|
||||
if post == nil || post.ID <= 0 || post.UserID <= 0 {
|
||||
return errs.ErrInvalidArgument
|
||||
}
|
||||
url := strings.TrimSpace(post.SourceURL)
|
||||
if url == "" {
|
||||
return errs.ErrInvalidArgument
|
||||
}
|
||||
post.SourceURL = url
|
||||
if post.Source == 0 {
|
||||
post.Source = entity.SourceBotRSS
|
||||
}
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if _, ok := m.botLinks[url]; ok {
|
||||
return errs.ErrDuplicate
|
||||
}
|
||||
if _, ok := m.posts[post.ID]; ok {
|
||||
return errs.ErrDuplicate
|
||||
}
|
||||
m.botLinks[url] = post.ID
|
||||
m.posts[post.ID] = clonePost(post)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MemoryRepo) SoftDeleteExpiredBotPosts(_ context.Context, nowMS int64, limit int) ([]*entity.Post, error) {
|
||||
if limit <= 0 {
|
||||
limit = 100
|
||||
}
|
||||
if nowMS <= 0 {
|
||||
nowMS = time.Now().UnixMilli()
|
||||
}
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
out := make([]*entity.Post, 0)
|
||||
for _, row := range m.posts {
|
||||
if row.Status != entity.StatusNormal || !row.Source.IsBot() {
|
||||
continue
|
||||
}
|
||||
if !row.Expired(nowMS) {
|
||||
continue
|
||||
}
|
||||
row.Status = entity.StatusDeleted
|
||||
out = append(out, clonePost(row))
|
||||
if len(out) >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (m *MemoryRepo) SoftDeletePost(_ context.Context, userID, postID int64) (*entity.Post, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
@ -22,6 +22,9 @@ type Post struct {
|
||||
AuditNote string `gorm:"column:audit_note;comment:拒绝原因"`
|
||||
AuditedAt *time.Time `gorm:"column:audited_at;comment:审核时间"`
|
||||
Status int16 `gorm:"column:status;comment:0正常 1已删"`
|
||||
Source int16 `gorm:"column:source;comment:0用户 1机器人海外媒体RSS"`
|
||||
SourceURL string `gorm:"column:source_url;comment:原文 URL"`
|
||||
ExpiresAt *time.Time `gorm:"column:expires_at;comment:机器人帖过期时间"`
|
||||
LikeCount int `gorm:"column:like_count;comment:点赞数"`
|
||||
CommentCount int `gorm:"column:comment_count;comment:评论数"`
|
||||
CreatedAt time.Time `gorm:"column:created_at;comment:发帖时间"`
|
||||
@ -37,8 +40,11 @@ func (r *Post) Entity() *entity.Post {
|
||||
ID: r.ID, UserID: r.UserID, Kind: entity.Kind(r.Kind), Text: r.Text, Content: r.Content,
|
||||
POI: entity.POI{Lat: r.POILat, Lng: r.POILng, Name: r.POIName, Address: r.POIAddress},
|
||||
Visibility: entity.Visibility(r.Visibility), AuditStatus: entity.AuditStatus(r.AuditStatus),
|
||||
Status: entity.Status(r.Status), LikeCount: r.LikeCount, CommentCount: r.CommentCount,
|
||||
CreatedAtMS: r.CreatedAt.UnixMilli(),
|
||||
Status: entity.Status(r.Status), Source: entity.Source(r.Source), SourceURL: r.SourceURL,
|
||||
LikeCount: r.LikeCount, CommentCount: r.CommentCount, CreatedAtMS: r.CreatedAt.UnixMilli(),
|
||||
}
|
||||
if r.ExpiresAt != nil {
|
||||
p.ExpiresAtMS = r.ExpiresAt.UnixMilli()
|
||||
}
|
||||
return p
|
||||
}
|
||||
@ -51,11 +57,15 @@ func postFrom(p *entity.Post) *Post {
|
||||
ID: p.ID, UserID: p.UserID, Kind: int16(p.Kind), Text: p.Text, Content: p.Content,
|
||||
POILat: p.POI.Lat, POILng: p.POI.Lng, POIName: p.POI.Name, POIAddress: p.POI.Address,
|
||||
Visibility: int16(p.Visibility), AuditStatus: int16(p.AuditStatus), Status: int16(p.Status),
|
||||
LikeCount: p.LikeCount, CommentCount: p.CommentCount,
|
||||
Source: int16(p.Source), SourceURL: p.SourceURL, LikeCount: p.LikeCount, CommentCount: p.CommentCount,
|
||||
}
|
||||
if p.CreatedAtMS > 0 {
|
||||
row.CreatedAt = time.UnixMilli(p.CreatedAtMS)
|
||||
}
|
||||
if p.ExpiresAtMS > 0 {
|
||||
t := time.UnixMilli(p.ExpiresAtMS)
|
||||
row.ExpiresAt = &t
|
||||
}
|
||||
if p.AuditStatus == entity.AuditApproved {
|
||||
t := row.CreatedAt
|
||||
if t.IsZero() {
|
||||
@ -66,6 +76,16 @@ func postFrom(p *entity.Post) *Post {
|
||||
return row
|
||||
}
|
||||
|
||||
// BotLink maps t_square_bot_links (RSS URL 去重).
|
||||
type BotLink struct {
|
||||
URLHash string `gorm:"column:url_hash;primaryKey;comment:URL SHA-256 hex"`
|
||||
URL string `gorm:"column:url;comment:原文 URL"`
|
||||
PostID int64 `gorm:"column:post_id;comment:帖 id"`
|
||||
CreatedAt time.Time `gorm:"column:created_at;comment:入库时间"`
|
||||
}
|
||||
|
||||
func (BotLink) TableName() string { return "t_square_bot_links" }
|
||||
|
||||
// Audience maps square_audience.
|
||||
type Audience struct {
|
||||
PostID int64 `gorm:"column:post_id;primaryKey;comment:帖 id"`
|
||||
|
||||
@ -3,12 +3,16 @@ package model
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"spark-server/internal/module/square/entity"
|
||||
)
|
||||
|
||||
// notExpiredSQL keeps user posts and unexpired bot posts.
|
||||
const notExpiredSQL = "(expires_at IS NULL OR expires_at > ?)"
|
||||
|
||||
func (m *Repo) GetPost(ctx context.Context, id int64) (*entity.Post, error) {
|
||||
db, err := m.session(ctx)
|
||||
if err != nil {
|
||||
@ -29,7 +33,7 @@ func (m *Repo) ListByAuthors(ctx context.Context, authorIDs []int64, cursor int6
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
q := db.Where("user_id IN ? AND status = ?", authorIDs, entity.StatusNormal)
|
||||
q := db.Where("user_id IN ? AND status = ? AND "+notExpiredSQL, authorIDs, entity.StatusNormal, time.Now())
|
||||
if cursor > 0 {
|
||||
q = q.Where("id < ?", cursor)
|
||||
}
|
||||
@ -50,9 +54,10 @@ func (m *Repo) ListPublicFeed(ctx context.Context, cursor int64, limit int) ([]*
|
||||
return nil, err
|
||||
}
|
||||
q := db.Where(
|
||||
"audit_status = ? AND status = ? AND visibility IN ?",
|
||||
"audit_status = ? AND status = ? AND visibility IN ? AND "+notExpiredSQL,
|
||||
entity.AuditApproved, entity.StatusNormal,
|
||||
[]int16{int16(entity.VisibilityPublic), int16(entity.VisibilityExclude)},
|
||||
time.Now(),
|
||||
)
|
||||
if cursor > 0 {
|
||||
q = q.Where("id < ?", cursor)
|
||||
|
||||
0
internal/module/squarebot/logic/.keep
Normal file
0
internal/module/squarebot/logic/.keep
Normal file
3
internal/module/squarebot/logic/doc.go
Normal file
3
internal/module/squarebot/logic/doc.go
Normal file
@ -0,0 +1,3 @@
|
||||
// Package logic is the square overseas-media bot worker (RSS → compress → square post).
|
||||
// Media helpers live in media.go so the RSS poller can compress before objectstore Put.
|
||||
package logic
|
||||
104
internal/module/squarebot/logic/media.go
Normal file
104
internal/module/squarebot/logic/media.go
Normal file
@ -0,0 +1,104 @@
|
||||
package logic
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
infmedia "spark-server/internal/infrastructure/media"
|
||||
pkgmedia "spark-server/internal/pkg/media"
|
||||
)
|
||||
|
||||
const maxDownloadBytes = 80 << 20 // refuse absurd remote blobs before compress
|
||||
|
||||
// MediaCompressor downloads remote media and compresses to App-aligned specs before objectstore write.
|
||||
// Square bot RSS worker should call FetchImage / FetchVideo instead of raw HTTP+Put.
|
||||
type MediaCompressor struct {
|
||||
Opt infmedia.Options
|
||||
Client *http.Client
|
||||
}
|
||||
|
||||
// NewMediaCompressor builds a compressor; ffmpegPath may be empty (defaults to "ffmpeg").
|
||||
func NewMediaCompressor(lim pkgmedia.Limits, jpegQuality, videoCRF int, ffmpegPath string) *MediaCompressor {
|
||||
return &MediaCompressor{
|
||||
Opt: infmedia.OptionsFromLimits(lim, jpegQuality, videoCRF, ffmpegPath),
|
||||
Client: &http.Client{
|
||||
Timeout: 60 * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// FetchImage GETs url and compresses to ≤1080p JPEG/PNG.
|
||||
func (c *MediaCompressor) FetchImage(ctx context.Context, url string) (*infmedia.BotAsset, error) {
|
||||
raw, err := c.download(ctx, url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return infmedia.PrepareBotImage(raw, c.Opt)
|
||||
}
|
||||
|
||||
// FetchAvatar GETs url and compresses to ≤200×200.
|
||||
func (c *MediaCompressor) FetchAvatar(ctx context.Context, url string) (*infmedia.BotAsset, error) {
|
||||
raw, err := c.download(ctx, url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return infmedia.PrepareBotAvatar(raw, c.Opt)
|
||||
}
|
||||
|
||||
// FetchVideo GETs url and compresses to ≤720p (requires ffmpeg).
|
||||
func (c *MediaCompressor) FetchVideo(ctx context.Context, url string) (*infmedia.BotAsset, error) {
|
||||
raw, err := c.download(ctx, url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return infmedia.PrepareBotVideo(ctx, raw, c.Opt)
|
||||
}
|
||||
|
||||
// CompressBytesImage compresses already-downloaded image bytes (e.g. RSS enclosure already fetched).
|
||||
func (c *MediaCompressor) CompressBytesImage(src []byte) (*infmedia.BotAsset, error) {
|
||||
return infmedia.PrepareBotImage(src, c.Opt)
|
||||
}
|
||||
|
||||
// CompressBytesVideo compresses already-downloaded video bytes.
|
||||
func (c *MediaCompressor) CompressBytesVideo(ctx context.Context, src []byte) (*infmedia.BotAsset, error) {
|
||||
return infmedia.PrepareBotVideo(ctx, src, c.Opt)
|
||||
}
|
||||
|
||||
func (c *MediaCompressor) download(ctx context.Context, url string) ([]byte, error) {
|
||||
url = strings.TrimSpace(url)
|
||||
if url == "" {
|
||||
return nil, fmt.Errorf("squarebot media: empty url")
|
||||
}
|
||||
client := c.Client
|
||||
if client == nil {
|
||||
client = http.DefaultClient
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("User-Agent", "SparkIM-SquareBot/1.0")
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = res.Body.Close() }()
|
||||
if res.StatusCode < 200 || res.StatusCode >= 300 {
|
||||
return nil, fmt.Errorf("squarebot media: http %d", res.StatusCode)
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(res.Body, maxDownloadBytes+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(data) > maxDownloadBytes {
|
||||
return nil, fmt.Errorf("squarebot media: download too large")
|
||||
}
|
||||
if len(data) == 0 {
|
||||
return nil, fmt.Errorf("squarebot media: empty body")
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
228
internal/module/squarebot/logic/rss.go
Normal file
228
internal/module/squarebot/logic/rss.go
Normal file
@ -0,0 +1,228 @@
|
||||
package logic
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"html"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
const (
|
||||
rssMaxBody = 2 << 20
|
||||
rssTimeout = 20 * time.Second
|
||||
summaryRunes = 280
|
||||
)
|
||||
|
||||
// FeedItem is one public RSS/Atom entry (title + summary + link).
|
||||
type FeedItem struct {
|
||||
Title string
|
||||
Summary string
|
||||
Link string
|
||||
Source string
|
||||
}
|
||||
|
||||
type rssFeed struct {
|
||||
Channel struct {
|
||||
Title string `xml:"title"`
|
||||
Items []rssItem `xml:"item"`
|
||||
} `xml:"channel"`
|
||||
}
|
||||
|
||||
type rssItem struct {
|
||||
Title string `xml:"title"`
|
||||
Link string `xml:"link"`
|
||||
Description string `xml:"description"`
|
||||
GUID string `xml:"guid"`
|
||||
}
|
||||
|
||||
type atomFeed struct {
|
||||
Title string `xml:"title"`
|
||||
Entries []atomEntry `xml:"entry"`
|
||||
}
|
||||
|
||||
type atomEntry struct {
|
||||
Title string `xml:"title"`
|
||||
Summary string `xml:"summary"`
|
||||
Content string `xml:"content"`
|
||||
Links []atomLink `xml:"link"`
|
||||
ID string `xml:"id"`
|
||||
}
|
||||
|
||||
type atomLink struct {
|
||||
Href string `xml:"href,attr"`
|
||||
Rel string `xml:"rel,attr"`
|
||||
}
|
||||
|
||||
var tagRe = regexp.MustCompile(`(?s)<[^>]*>`)
|
||||
|
||||
// FetchFeed loads and parses a public RSS 2.0 or Atom URL.
|
||||
func FetchFeed(ctx context.Context, client *http.Client, feedURL string) ([]FeedItem, error) {
|
||||
feedURL = strings.TrimSpace(feedURL)
|
||||
if feedURL == "" {
|
||||
return nil, fmt.Errorf("empty feed url")
|
||||
}
|
||||
if client == nil {
|
||||
client = &http.Client{Timeout: rssTimeout}
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, feedURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("User-Agent", "SparkIM-SquareBot/1.0 (+https://spark.wyrs365.com)")
|
||||
req.Header.Set("Accept", "application/rss+xml, application/atom+xml, application/xml, text/xml, */*")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return nil, fmt.Errorf("feed status %d", resp.StatusCode)
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, rssMaxBody))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return parseFeed(body, feedURL)
|
||||
}
|
||||
|
||||
func parseFeed(body []byte, feedURL string) ([]FeedItem, error) {
|
||||
var rss rssFeed
|
||||
if err := xml.Unmarshal(body, &rss); err == nil && len(rss.Channel.Items) > 0 {
|
||||
src := strings.TrimSpace(rss.Channel.Title)
|
||||
out := make([]FeedItem, 0, len(rss.Channel.Items))
|
||||
for _, it := range rss.Channel.Items {
|
||||
link := firstNonEmpty(strings.TrimSpace(it.Link), strings.TrimSpace(it.GUID))
|
||||
item := FeedItem{
|
||||
Title: cleanText(it.Title),
|
||||
Summary: cleanText(it.Description),
|
||||
Link: normalizeURL(link),
|
||||
Source: src,
|
||||
}
|
||||
if item.Title == "" || item.Link == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, item)
|
||||
}
|
||||
if len(out) > 0 {
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
var atom atomFeed
|
||||
if err := xml.Unmarshal(body, &atom); err != nil {
|
||||
return nil, fmt.Errorf("parse feed %s: %w", feedURL, err)
|
||||
}
|
||||
src := cleanText(atom.Title)
|
||||
out := make([]FeedItem, 0, len(atom.Entries))
|
||||
for _, e := range atom.Entries {
|
||||
link := atomBestLink(e.Links)
|
||||
if link == "" {
|
||||
link = strings.TrimSpace(e.ID)
|
||||
}
|
||||
sum := firstNonEmpty(e.Summary, e.Content)
|
||||
item := FeedItem{
|
||||
Title: cleanText(e.Title),
|
||||
Summary: cleanText(sum),
|
||||
Link: normalizeURL(link),
|
||||
Source: src,
|
||||
}
|
||||
if item.Title == "" || item.Link == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, item)
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil, fmt.Errorf("empty feed %s", feedURL)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func atomBestLink(links []atomLink) string {
|
||||
var alt, self, any string
|
||||
for _, l := range links {
|
||||
href := strings.TrimSpace(l.Href)
|
||||
if href == "" {
|
||||
continue
|
||||
}
|
||||
any = href
|
||||
switch strings.ToLower(strings.TrimSpace(l.Rel)) {
|
||||
case "", "alternate":
|
||||
if alt == "" {
|
||||
alt = href
|
||||
}
|
||||
case "self":
|
||||
if self == "" {
|
||||
self = href
|
||||
}
|
||||
}
|
||||
}
|
||||
return firstNonEmpty(alt, self, any)
|
||||
}
|
||||
|
||||
func FormatPostText(item FeedItem) string {
|
||||
title := strings.TrimSpace(item.Title)
|
||||
sum := truncateRunes(strings.TrimSpace(item.Summary), summaryRunes)
|
||||
link := strings.TrimSpace(item.Link)
|
||||
src := strings.TrimSpace(item.Source)
|
||||
var b strings.Builder
|
||||
if src != "" {
|
||||
b.WriteString(src)
|
||||
b.WriteString("\n")
|
||||
}
|
||||
b.WriteString(title)
|
||||
if sum != "" && !strings.EqualFold(sum, title) {
|
||||
b.WriteString("\n\n")
|
||||
b.WriteString(sum)
|
||||
}
|
||||
b.WriteString("\n\n")
|
||||
b.WriteString(link)
|
||||
text := strings.TrimSpace(b.String())
|
||||
return truncateRunes(text, 1000)
|
||||
}
|
||||
|
||||
func cleanText(s string) string {
|
||||
s = html.UnescapeString(s)
|
||||
s = tagRe.ReplaceAllString(s, " ")
|
||||
s = strings.Join(strings.Fields(s), " ")
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
|
||||
func truncateRunes(s string, max int) string {
|
||||
if max <= 0 || utf8.RuneCountInString(s) <= max {
|
||||
return s
|
||||
}
|
||||
runes := []rune(s)
|
||||
return string(runes[:max-1]) + "…"
|
||||
}
|
||||
|
||||
func normalizeURL(raw string) string {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return ""
|
||||
}
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil || u.Scheme == "" || u.Host == "" {
|
||||
return ""
|
||||
}
|
||||
scheme := strings.ToLower(u.Scheme)
|
||||
if scheme != "http" && scheme != "https" {
|
||||
return ""
|
||||
}
|
||||
u.Fragment = ""
|
||||
return u.String()
|
||||
}
|
||||
|
||||
func firstNonEmpty(vals ...string) string {
|
||||
for _, v := range vals {
|
||||
if strings.TrimSpace(v) != "" {
|
||||
return strings.TrimSpace(v)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
213
internal/module/squarebot/logic/service.go
Normal file
213
internal/module/squarebot/logic/service.go
Normal file
@ -0,0 +1,213 @@
|
||||
package logic
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
squarelogic "spark-server/internal/module/square/logic"
|
||||
userentity "spark-server/internal/module/user/entity"
|
||||
"spark-server/internal/pkg/errs"
|
||||
)
|
||||
|
||||
// Options configures the square media bot worker.
|
||||
type Options struct {
|
||||
Enabled bool
|
||||
Interval time.Duration
|
||||
TTL time.Duration
|
||||
Feeds []string
|
||||
BotIndexes []int
|
||||
HTTPClient *http.Client
|
||||
}
|
||||
|
||||
// Users resolves seeded robot accounts.
|
||||
type Users interface {
|
||||
GetByEmail(ctx context.Context, email string) (*userentity.User, error)
|
||||
}
|
||||
|
||||
// Square publishes and expires bot posts.
|
||||
type Square interface {
|
||||
PublishBotMedia(ctx context.Context, userID int64, in *squarelogic.BotPublishInput) (*squarelogic.View, error)
|
||||
ExpireBotPosts(ctx context.Context, now time.Time, limit int) (int, error)
|
||||
}
|
||||
|
||||
// Locker is an optional distributed tick lock (e.g. Redis SETNX).
|
||||
type Locker interface {
|
||||
TryLock(ctx context.Context, key string, ttl time.Duration) (bool, error)
|
||||
}
|
||||
|
||||
// Service runs one RSS publish + TTL cleanup tick.
|
||||
type Service struct {
|
||||
log *zap.Logger
|
||||
square Square
|
||||
users Users
|
||||
locker Locker
|
||||
opts Options
|
||||
client *http.Client
|
||||
rand *rand.Rand
|
||||
lockKey string
|
||||
}
|
||||
|
||||
func NewService(log *zap.Logger, square Square, users Users, locker Locker, opts Options) *Service {
|
||||
if log == nil {
|
||||
log = zap.NewNop()
|
||||
}
|
||||
opts = normalizeOptions(opts)
|
||||
client := opts.HTTPClient
|
||||
if client == nil {
|
||||
client = &http.Client{Timeout: rssTimeout}
|
||||
}
|
||||
return &Service{
|
||||
log: log, square: square, users: users, locker: locker, opts: opts,
|
||||
client: client, rand: rand.New(rand.NewSource(time.Now().UnixNano())),
|
||||
lockKey: "squarebot:tick",
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeOptions(o Options) Options {
|
||||
if o.Interval <= 0 {
|
||||
o.Interval = 5 * time.Minute
|
||||
}
|
||||
if o.TTL <= 0 {
|
||||
o.TTL = 15 * 24 * time.Hour
|
||||
}
|
||||
if len(o.Feeds) == 0 {
|
||||
o.Feeds = DefaultFeeds()
|
||||
}
|
||||
if len(o.BotIndexes) == 0 {
|
||||
o.BotIndexes = defaultBotIndexes()
|
||||
}
|
||||
return o
|
||||
}
|
||||
|
||||
// DefaultFeeds are public overseas mainstream media RSS endpoints (title/summary/link only).
|
||||
func DefaultFeeds() []string {
|
||||
return []string{
|
||||
"https://feeds.bbci.co.uk/news/world/rss.xml",
|
||||
"https://www.theguardian.com/world/rss",
|
||||
"https://feeds.npr.org/1004/rss.xml",
|
||||
"https://www.aljazeera.com/xml/rss/all.xml",
|
||||
"https://rss.nytimes.com/services/xml/rss/nyt/World.xml",
|
||||
"https://www.reutersagency.com/feed/?best-topics=world-including-us&post_type=best",
|
||||
}
|
||||
}
|
||||
|
||||
func defaultBotIndexes() []int {
|
||||
out := make([]int, 0, userentity.BotCount)
|
||||
for i := 1; i <= userentity.BotCount; i++ {
|
||||
out = append(out, i)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (s *Service) Enabled() bool {
|
||||
return s != nil && s.opts.Enabled && s.square != nil && s.users != nil
|
||||
}
|
||||
|
||||
func (s *Service) Interval() time.Duration {
|
||||
if s == nil {
|
||||
return 5 * time.Minute
|
||||
}
|
||||
return s.opts.Interval
|
||||
}
|
||||
|
||||
// Tick expires old bot posts then publishes one new RSS item via a random bot.
|
||||
func (s *Service) Tick(ctx context.Context) error {
|
||||
if !s.Enabled() {
|
||||
return nil
|
||||
}
|
||||
if s.locker != nil {
|
||||
ok, err := s.locker.TryLock(ctx, s.lockKey, s.opts.Interval)
|
||||
if err != nil {
|
||||
s.log.Warn("squarebot lock", zap.Error(err))
|
||||
} else if !ok {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if n, err := s.square.ExpireBotPosts(ctx, time.Now(), 200); err != nil {
|
||||
s.log.Warn("squarebot expire", zap.Error(err))
|
||||
} else if n > 0 {
|
||||
s.log.Info("squarebot expired", zap.Int("count", n))
|
||||
}
|
||||
bot, err := s.pickBot(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
item, err := s.pickItem(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
text := FormatPostText(item)
|
||||
view, err := s.square.PublishBotMedia(ctx, bot.ID, &squarelogic.BotPublishInput{
|
||||
Text: text, SourceURL: item.Link, TTL: s.opts.TTL,
|
||||
})
|
||||
if err != nil {
|
||||
if errors.Is(err, errs.ErrDuplicate) {
|
||||
s.log.Info("squarebot skip duplicate", zap.String("url", item.Link))
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
s.log.Info("squarebot posted",
|
||||
zap.Int64("bot_uid", bot.ID),
|
||||
zap.String("bot", bot.Nickname),
|
||||
zap.Int64("post_id", view.ID),
|
||||
zap.String("url", item.Link),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) pickBot(ctx context.Context) (*userentity.User, error) {
|
||||
indexes := append([]int(nil), s.opts.BotIndexes...)
|
||||
s.rand.Shuffle(len(indexes), func(i, j int) { indexes[i], indexes[j] = indexes[j], indexes[i] })
|
||||
var last error
|
||||
for _, idx := range indexes {
|
||||
email := userentity.BotEmail(idx)
|
||||
if email == "" {
|
||||
continue
|
||||
}
|
||||
u, err := s.users.GetByEmail(ctx, email)
|
||||
if err != nil {
|
||||
last = err
|
||||
continue
|
||||
}
|
||||
if u != nil && u.ID > 0 {
|
||||
return u, nil
|
||||
}
|
||||
}
|
||||
if last != nil {
|
||||
return nil, last
|
||||
}
|
||||
return nil, errs.ErrNotFound
|
||||
}
|
||||
|
||||
func (s *Service) pickItem(ctx context.Context) (FeedItem, error) {
|
||||
feeds := append([]string(nil), s.opts.Feeds...)
|
||||
s.rand.Shuffle(len(feeds), func(i, j int) { feeds[i], feeds[j] = feeds[j], feeds[i] })
|
||||
var last error
|
||||
for _, feed := range feeds {
|
||||
feed = strings.TrimSpace(feed)
|
||||
if feed == "" {
|
||||
continue
|
||||
}
|
||||
items, err := FetchFeed(ctx, s.client, feed)
|
||||
if err != nil {
|
||||
last = err
|
||||
s.log.Warn("squarebot feed", zap.String("feed", feed), zap.Error(err))
|
||||
continue
|
||||
}
|
||||
if len(items) == 0 {
|
||||
continue
|
||||
}
|
||||
return items[s.rand.Intn(len(items))], nil
|
||||
}
|
||||
if last != nil {
|
||||
return FeedItem{}, last
|
||||
}
|
||||
return FeedItem{}, errs.ErrNotFound
|
||||
}
|
||||
@ -5,7 +5,10 @@ import (
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"spark-server/internal/pkg/media"
|
||||
"spark-server/internal/pkg/ratelimit"
|
||||
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// Presigner issues short-lived object-storage URLs. Infrastructure implements it.
|
||||
@ -23,6 +26,8 @@ type Options struct {
|
||||
Limit *ratelimit.Limiter
|
||||
Now func() time.Time
|
||||
Groups AvatarACL
|
||||
Media media.Policy
|
||||
Log *zap.Logger
|
||||
}
|
||||
|
||||
// Service issues upload/download credentials. HTTP and TCP share it.
|
||||
@ -32,6 +37,8 @@ type Service struct {
|
||||
limit *ratelimit.Limiter
|
||||
now func() time.Time
|
||||
groups AvatarACL
|
||||
media media.Policy
|
||||
log *zap.Logger
|
||||
}
|
||||
|
||||
func NewService(store Presigner, opt Options) *Service {
|
||||
@ -43,7 +50,11 @@ func NewService(store Presigner, opt Options) *Service {
|
||||
if clock == nil {
|
||||
clock = func() time.Time { return time.Now().UTC() }
|
||||
}
|
||||
return &Service{store: store, ttl: ttl, limit: opt.Limit, now: clock, groups: opt.Groups}
|
||||
log := opt.Log
|
||||
if log == nil {
|
||||
log = zap.NewNop()
|
||||
}
|
||||
return &Service{store: store, ttl: ttl, limit: opt.Limit, now: clock, groups: opt.Groups, media: opt.Media, log: log}
|
||||
}
|
||||
|
||||
func (s *Service) ready() bool {
|
||||
|
||||
@ -7,6 +7,7 @@ var (
|
||||
ErrFrameTooLarge = NewError("40002", "frame too large", "")
|
||||
ErrInvalidFrame = NewError("40003", "invalid frame", "")
|
||||
ErrProxyHeader = NewError("40004", "invalid proxy protocol header", "")
|
||||
ErrMediaOversized = NewError("40005", "media oversized", "")
|
||||
ErrUnauthorized = NewError("40101", "unauthorized", "")
|
||||
ErrNotAuthenticated = NewError("40102", "not authenticated", "")
|
||||
ErrVerifyCodeInvalid = NewError("40103", "verification code invalid", "")
|
||||
|
||||
@ -17,6 +17,9 @@ func init() {
|
||||
"err.40004": {
|
||||
ZhCN: "代理协议错误", En: "invalid proxy protocol header", ZhTW: "代理協定錯誤",
|
||||
},
|
||||
"err.40005": {
|
||||
ZhCN: "媒体过大或分辨率超限", En: "media oversized", ZhTW: "媒體過大或解析度超限",
|
||||
},
|
||||
"err.40101": {
|
||||
ZhCN: "未授权", En: "unauthorized", ZhTW: "未授權",
|
||||
},
|
||||
|
||||
@ -62,6 +62,11 @@ func init() {
|
||||
KeyUploadDownloadItemsInvalid: {
|
||||
ZhCN: "一次最多签发 9 个下载凭证", En: "At most 9 download credentials per request", ZhTW: "一次最多簽發 9 個下載憑證",
|
||||
},
|
||||
KeyMediaOversized: {
|
||||
ZhCN: "图片最长边须≤1080、视频≤720且体积不超限(头像≤200×200);请先在客户端压缩后再上传",
|
||||
En: "Image longest edge must be ≤1080, video ≤720 within size limits (avatar ≤200×200); compress on client before upload",
|
||||
ZhTW: "圖片最長邊須≤1080、影片≤720且體積不超限(頭像≤200×200);請先在客戶端壓縮後再上傳",
|
||||
},
|
||||
KeyFavoriteMsgRequired: {
|
||||
ZhCN: "请选择消息", En: "Please choose a message", ZhTW: "請選擇訊息",
|
||||
},
|
||||
|
||||
@ -102,6 +102,7 @@ const (
|
||||
KeyUploadItemsRequired = "validate.upload.items_required"
|
||||
KeyUploadItemsInvalid = "validate.upload.items_invalid"
|
||||
KeyUploadDownloadItemsInvalid = "validate.upload.download_items_invalid"
|
||||
KeyMediaOversized = "err.media_oversized"
|
||||
KeyFavoriteMsgRequired = "validate.favorite.msg_required"
|
||||
KeyFavoriteMsgInvalid = "validate.favorite.msg_invalid"
|
||||
KeyMessageClientMsgRequired = "validate.message.client_msg_required"
|
||||
|
||||
83
internal/pkg/media/check.go
Normal file
83
internal/pkg/media/check.go
Normal file
@ -0,0 +1,83 @@
|
||||
package media
|
||||
|
||||
import "fmt"
|
||||
|
||||
// Kind classifies media for policy checks.
|
||||
type Kind string
|
||||
|
||||
const (
|
||||
KindImage Kind = "image"
|
||||
KindVideo Kind = "video"
|
||||
KindAvatar Kind = "avatar"
|
||||
)
|
||||
|
||||
// Violation describes an oversize resource (dimensions and/or bytes).
|
||||
type Violation struct {
|
||||
Kind Kind
|
||||
Reason string
|
||||
Width int
|
||||
Height int
|
||||
MaxEdge int
|
||||
Size int64
|
||||
MaxSize int64
|
||||
Duration int // ms; video only
|
||||
}
|
||||
|
||||
func (v *Violation) Error() string {
|
||||
if v == nil {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("media %s oversize: %s", v.Kind, v.Reason)
|
||||
}
|
||||
|
||||
// Policy controls user-path detection (bots always compress via infrastructure/media).
|
||||
type Policy struct {
|
||||
Enabled bool // master switch for user checks
|
||||
UserReject bool // when true, publish/upload returns ErrMediaOversized; else log-only
|
||||
Limits Limits
|
||||
}
|
||||
|
||||
// CheckImageMeta validates declared image width/height/size against Limits.
|
||||
// Zero width/height skips dimension check (client omitted); size 0 skips byte check.
|
||||
func (p Policy) CheckImageMeta(w, h int, size int64) *Violation {
|
||||
return p.check(KindImage, w, h, size, 0)
|
||||
}
|
||||
|
||||
// CheckVideoMeta validates declared video metadata.
|
||||
func (p Policy) CheckVideoMeta(w, h int, size int64, durationMS int) *Violation {
|
||||
return p.check(KindVideo, w, h, size, durationMS)
|
||||
}
|
||||
|
||||
// CheckAvatarMeta validates avatar dimensions/size (≤200×200 by default).
|
||||
func (p Policy) CheckAvatarMeta(w, h int, size int64) *Violation {
|
||||
return p.check(KindAvatar, w, h, size, 0)
|
||||
}
|
||||
|
||||
func (p Policy) check(kind Kind, w, h int, size int64, durationMS int) *Violation {
|
||||
if !p.Enabled {
|
||||
return nil
|
||||
}
|
||||
lim := p.Limits.Normalize()
|
||||
maxEdge, maxSize := lim.ImageMaxEdge, lim.ImageMaxBytes
|
||||
switch kind {
|
||||
case KindVideo:
|
||||
maxEdge, maxSize = lim.VideoMaxEdge, lim.VideoMaxBytes
|
||||
case KindAvatar:
|
||||
maxEdge, maxSize = lim.AvatarMaxEdge, lim.AvatarMaxBytes
|
||||
}
|
||||
if w > 0 && h > 0 && NeedsResize(w, h, maxEdge) {
|
||||
return &Violation{
|
||||
Kind: kind, Reason: "edge_exceeded",
|
||||
Width: w, Height: h, MaxEdge: maxEdge,
|
||||
Size: size, MaxSize: maxSize, Duration: durationMS,
|
||||
}
|
||||
}
|
||||
if size > 0 && maxSize > 0 && size > maxSize {
|
||||
return &Violation{
|
||||
Kind: kind, Reason: "size_exceeded",
|
||||
Width: w, Height: h, MaxEdge: maxEdge,
|
||||
Size: size, MaxSize: maxSize, Duration: durationMS,
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
45
internal/pkg/media/check_test.go
Normal file
45
internal/pkg/media/check_test.go
Normal file
@ -0,0 +1,45 @@
|
||||
package media
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestCheckImageMeta(t *testing.T) {
|
||||
p := Policy{Enabled: true, Limits: DefaultLimits()}
|
||||
if v := p.CheckImageMeta(1080, 720, 1<<20); v != nil {
|
||||
t.Fatalf("expected ok, got %+v", v)
|
||||
}
|
||||
if v := p.CheckImageMeta(2000, 1000, 1<<20); v == nil || v.Reason != "edge_exceeded" {
|
||||
t.Fatalf("want edge_exceeded, got %+v", v)
|
||||
}
|
||||
if v := p.CheckImageMeta(800, 600, 20<<20); v == nil || v.Reason != "size_exceeded" {
|
||||
t.Fatalf("want size_exceeded, got %+v", v)
|
||||
}
|
||||
p.Enabled = false
|
||||
if v := p.CheckImageMeta(4000, 4000, 50<<20); v != nil {
|
||||
t.Fatalf("disabled should skip, got %+v", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckVideoAndAvatar(t *testing.T) {
|
||||
p := Policy{Enabled: true, Limits: DefaultLimits()}
|
||||
if v := p.CheckVideoMeta(1280, 720, 10<<20, 5000); v == nil || v.Reason != "edge_exceeded" {
|
||||
t.Fatalf("1280 edge should fail vs 720: %+v", v)
|
||||
}
|
||||
if v := p.CheckVideoMeta(720, 480, 10<<20, 5000); v != nil {
|
||||
t.Fatalf("720p ok: %+v", v)
|
||||
}
|
||||
if v := p.CheckAvatarMeta(200, 200, 100<<10); v != nil {
|
||||
t.Fatalf("avatar ok: %+v", v)
|
||||
}
|
||||
if v := p.CheckAvatarMeta(400, 400, 100<<10); v == nil {
|
||||
t.Fatal("avatar 400 should fail")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNeedsResize(t *testing.T) {
|
||||
if NeedsResize(1080, 720, 1080) {
|
||||
t.Fatal("equal edge should not resize")
|
||||
}
|
||||
if !NeedsResize(1081, 1, 1080) {
|
||||
t.Fatal("want resize")
|
||||
}
|
||||
}
|
||||
77
internal/pkg/media/limits.go
Normal file
77
internal/pkg/media/limits.go
Normal file
@ -0,0 +1,77 @@
|
||||
// Package media holds shared media size/resolution policy (no I/O).
|
||||
// Client App compresses user uploads; bots must compress server-side via infrastructure/media.
|
||||
package media
|
||||
|
||||
// Default limits aligned with App frontend (image ≤1080p, video ≤720p ≤100MB, avatar ≤200×200).
|
||||
const (
|
||||
DefaultImageMaxEdge = 1080
|
||||
DefaultVideoMaxEdge = 720
|
||||
DefaultAvatarMaxEdge = 200
|
||||
DefaultImageMaxBytes = 10 << 20 // 10 MiB (square/moment image purpose)
|
||||
DefaultVideoMaxBytes = 100 << 20 // 100 MiB
|
||||
DefaultAvatarMaxBytes = 2 << 20 // 2 MiB (upload purpose hard cap)
|
||||
DefaultJPEGQuality = 85
|
||||
DefaultVideoCRF = 23
|
||||
)
|
||||
|
||||
// Limits is the resolution/size policy shared by user checks and bot compress.
|
||||
type Limits struct {
|
||||
ImageMaxEdge int
|
||||
VideoMaxEdge int
|
||||
AvatarMaxEdge int
|
||||
ImageMaxBytes int64
|
||||
VideoMaxBytes int64
|
||||
AvatarMaxBytes int64
|
||||
}
|
||||
|
||||
// DefaultLimits returns client-aligned defaults.
|
||||
func DefaultLimits() Limits {
|
||||
return Limits{
|
||||
ImageMaxEdge: DefaultImageMaxEdge,
|
||||
VideoMaxEdge: DefaultVideoMaxEdge,
|
||||
AvatarMaxEdge: DefaultAvatarMaxEdge,
|
||||
ImageMaxBytes: DefaultImageMaxBytes,
|
||||
VideoMaxBytes: DefaultVideoMaxBytes,
|
||||
AvatarMaxBytes: DefaultAvatarMaxBytes,
|
||||
}
|
||||
}
|
||||
|
||||
// Normalize fills zero fields with defaults.
|
||||
func (l Limits) Normalize() Limits {
|
||||
d := DefaultLimits()
|
||||
if l.ImageMaxEdge <= 0 {
|
||||
l.ImageMaxEdge = d.ImageMaxEdge
|
||||
}
|
||||
if l.VideoMaxEdge <= 0 {
|
||||
l.VideoMaxEdge = d.VideoMaxEdge
|
||||
}
|
||||
if l.AvatarMaxEdge <= 0 {
|
||||
l.AvatarMaxEdge = d.AvatarMaxEdge
|
||||
}
|
||||
if l.ImageMaxBytes <= 0 {
|
||||
l.ImageMaxBytes = d.ImageMaxBytes
|
||||
}
|
||||
if l.VideoMaxBytes <= 0 {
|
||||
l.VideoMaxBytes = d.VideoMaxBytes
|
||||
}
|
||||
if l.AvatarMaxBytes <= 0 {
|
||||
l.AvatarMaxBytes = d.AvatarMaxBytes
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
||||
// LongestEdge returns max(w, h).
|
||||
func LongestEdge(w, h int) int {
|
||||
if w > h {
|
||||
return w
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// NeedsResize reports whether the frame exceeds maxEdge (already-smaller is kept as-is).
|
||||
func NeedsResize(w, h, maxEdge int) bool {
|
||||
if maxEdge <= 0 || w <= 0 || h <= 0 {
|
||||
return false
|
||||
}
|
||||
return LongestEdge(w, h) > maxEdge
|
||||
}
|
||||
34
internal/pkg/media/publish.go
Normal file
34
internal/pkg/media/publish.go
Normal file
@ -0,0 +1,34 @@
|
||||
package media
|
||||
|
||||
// ImageMeta is client-declared image metadata on publish.
|
||||
type ImageMeta struct {
|
||||
Width int
|
||||
Height int
|
||||
Size int64
|
||||
}
|
||||
|
||||
// VideoMeta is client-declared video metadata on publish.
|
||||
type VideoMeta struct {
|
||||
Width int
|
||||
Height int
|
||||
Size int64
|
||||
DurationMS int
|
||||
}
|
||||
|
||||
// CheckPublishImages returns the first image violation, if any.
|
||||
func (p Policy) CheckPublishImages(images []ImageMeta) *Violation {
|
||||
for _, img := range images {
|
||||
if v := p.CheckImageMeta(img.Width, img.Height, img.Size); v != nil {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CheckPublishVideo returns a video violation, if any.
|
||||
func (p Policy) CheckPublishVideo(v *VideoMeta) *Violation {
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
return p.CheckVideoMeta(v.Width, v.Height, v.Size, v.DurationMS)
|
||||
}
|
||||
19
migrations/0026_square_bot_rss.sql
Normal file
19
migrations/0026_square_bot_rss.sql
Normal file
@ -0,0 +1,19 @@
|
||||
-- 广场机器人海外媒体 RSS:帖来源、原文 URL、过期时间与 URL 去重。
|
||||
|
||||
ALTER TABLE t_square_posts
|
||||
ADD COLUMN source SMALLINT NOT NULL DEFAULT 0 COMMENT '0用户 1机器人海外媒体RSS' AFTER status,
|
||||
ADD COLUMN source_url VARCHAR(768) NOT NULL DEFAULT '' COMMENT '原文 URL,机器人帖去重键' AFTER source,
|
||||
ADD COLUMN expires_at DATETIME(3) NULL COMMENT '机器人帖过期时间;到期不可见并清理' AFTER source_url;
|
||||
|
||||
ALTER TABLE t_square_posts
|
||||
ADD KEY idx_square_posts_expires (expires_at, status),
|
||||
ADD KEY idx_square_posts_source_url (source_url);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS t_square_bot_links (
|
||||
url_hash CHAR(64) NOT NULL COMMENT '规范化 URL 的 SHA-256 hex',
|
||||
url VARCHAR(768) NOT NULL COMMENT '原文 URL',
|
||||
post_id BIGINT NOT NULL COMMENT '关联 t_square_posts.id',
|
||||
created_at DATETIME(3) NOT NULL COMMENT '入库时间',
|
||||
PRIMARY KEY (url_hash),
|
||||
KEY idx_square_bot_links_post (post_id)
|
||||
) COMMENT='广场机器人 RSS 原文 URL 去重';
|
||||
Loading…
Reference in New Issue
Block a user