Spark/spark-server
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: download url

Browse Source
This commit is contained in:
zhr 2026-09-18 18:27:49 +08:00
parent 0c821ffa16
commit bebf76e582
15 changed files with 294 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
README.md
View File

@ -202,8 +202,8 @@ Java / Swift 示例：`docs/clients/`，OpenAPI 里也可点开。
| purpose | object_key | 生命周期 |
| --- | --- | --- |
| `avatar` | `perm/avatar/{yyyyMMdd}/{name}.{ext}` | 永久 |
| `group_avatar` | `perm/group/{群号}/{name}.{ext}` | 永久 |
| `avatar` | `perm/avatar/{yyyyMMdd}/{name}.{ext}` | 永久；匿名 GET |
| `group_avatar` | `perm/group/{群号}/{name}.{ext}` | 永久；匿名 GET |
| `chat_image` | `chat/image/{yyyyMMdd}/{name}.{ext}` | 桶生命周期 |
| `chat_voice` | `chat/voice/{yyyyMMdd}/{name}.{ext}` | 桶生命周期 |
| `chat_video` | `chat/video/{yyyyMMdd}/{name}.{ext}` | 桶生命周期 |
@ -217,6 +217,8 @@ Java / Swift 示例：`docs/clients/`，OpenAPI 里也可点开。
合法 key：以 `perm/` 或 `chat/` 开头；禁止 `..`、空白、`\`、前导 `/`；最长 512。
头像匿名读：`perm/avatar/`、`perm/group/` 可直接 GET `{public_endpoint}/spark/{key}`（Nginx 也可省略 `/spark`）。聊天、收藏、朋友圈仍须 `POST /v1/files/downloads`。
发图片：客户端只 PUT 原图，再 `POST /v1/files/variants` 让服务端出缩略图和预览。消息里只存 key。
进程访问 RustFS 用内网 `object_storage.endpoint`（Docker 里是 `rustfs:9000`），签给客户端的 URL 用 `public_endpoint`（`https://sparkr.wyrs365.com`）。

1
deploy/nginx/README.md
View File

@ -27,6 +27,7 @@ spark-server (Docker):
- Image / config / logs: `/data/spark-server/{images,configs,logs}`
- Public security group: 443 only
- Clients upload/download at `https://sparkr.wyrs365.com` (path-style). spark-server talks to `rustfs:9000` on the Docker network and signs URLs with `object_storage.public_endpoint`.
- Unsigned GET is allowed only for `/perm/avatar/` and `/perm/group/` (rewritten to `/spark/...`). Chat and other prefixes stay SigV4.
```bash
bash deploy/nginx/install-host.sh

27
deploy/nginx/host/sparkr.conf
View File

@ -1,7 +1,8 @@
# Spark RustFS S3 — sparkr.wyrs365.com
# Public :443 is stream ssl_preread (see stream.d/spark-sni.conf).
# TLS terminates on loopback :8443; proxy HTTP to Docker rustfs on :9000.
# Do not rewrite the URI: SigV4 signs host + path.
# Signed PUT/GET keep the path as-is (/spark/...). Unsigned avatar/group
# URLs omit the bucket; only those prefixes are rewritten and publicly readable.
upstream spark_rustfs {
server 127.0.0.1:9000;
@ -46,6 +47,30 @@ server {
proxy_http_version 1.1;
chunked_transfer_encoding off;
# Path-style S3 is /{bucket}/{key}. Clients often GET {public}/{object_key}.
# Rewrite only public prefixes; chat/ and perm/fav/ stay 403 without SigV4.
location ^~ /perm/avatar/ {
rewrite ^ /spark$request_uri break;
proxy_pass http://spark_rustfs;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto https;
proxy_set_header Connection "";
proxy_cache_convert_head off;
}
location ^~ /perm/group/ {
rewrite ^ /spark$request_uri break;
proxy_pass http://spark_rustfs;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto https;
proxy_set_header Connection "";
proxy_cache_convert_head off;
}
location / {
proxy_pass http://spark_rustfs;
proxy_set_header Host $http_host;

45
internal/infrastructure/objectstore/policy.go
View File

@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
@ -23,6 +24,7 @@ func (s *Store) ensureBucket(ctx context.Context, region string, chatExpireDays
}
s.applyLifecycle(ctx, chatExpireDays)
s.applyCORS(ctx)
s.applyPublicRead(ctx)
return nil
}
@ -83,3 +85,46 @@ func (s *Store) applyCORS(ctx context.Context) {
s.log.Warn("objectstore cors", zap.Error(err))
}
}
// applyPublicRead lets browsers load avatars without a presigned GET.
// chat/ and perm/fav/ stay private.
func (s *Store) applyPublicRead(ctx context.Context) {
pabCtx, cancel := context.WithTimeout(ctx, 3*time.Second)
defer cancel()
_, err := s.client.PutPublicAccessBlock(pabCtx, &s3.PutPublicAccessBlockInput{
Bucket: aws.String(s.bucket),
PublicAccessBlockConfiguration: &types.PublicAccessBlockConfiguration{
BlockPublicAcls: aws.Bool(false),
IgnorePublicAcls: aws.Bool(false),
BlockPublicPolicy: aws.Bool(false),
RestrictPublicBuckets: aws.Bool(false),
},
})
if err != nil {
s.log.Warn("objectstore public access block", zap.Error(err))
}
policy := publicReadPolicy(s.bucket)
_, err = s.client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
Bucket: aws.String(s.bucket),
Policy: aws.String(policy),
})
if err != nil {
s.log.Warn("objectstore public policy", zap.Error(err))
}
}
func publicReadPolicy(bucket string) string {
return fmt.Sprintf(`{
"Version": "2012-10-17",
"Statement": [{
"Sid": "PublicReadAvatars",
"Effect": "Allow",
"Principal": "*",
"Action": ["s3:GetObject"],
"Resource": [
"arn:aws:s3:::%[1]s/perm/avatar/*",
"arn:aws:s3:::%[1]s/perm/group/*"
]
}]
}`, bucket)
}

10
internal/infrastructure/objectstore/store_test.go
View File

@ -104,6 +104,16 @@ func TestGetPutNotReady(t *testing.T) {
}
}
func TestPublicReadPolicyCoversAvatarsOnly(t *testing.T) {
p := publicReadPolicy("spark")
if !strings.Contains(p, "spark/perm/avatar/*") || !strings.Contains(p, "spark/perm/group/*") {
t.Fatalf("%s", p)
}
if strings.Contains(p, "chat/") || strings.Contains(p, "perm/fav") || strings.Contains(p, "perm/moment") {
t.Fatalf("chat/fav/moment must stay private: %s", p)
}
}
func TestMapObjectErr(t *testing.T) {
err := mapObjectErr(&types.NoSuchKey{Message: aws.String("gone")})
if !errors.Is(err, errs.ErrMediaExpired) {

5
internal/module/upload/entity/credential.go
View File

@ -20,3 +20,8 @@ type Download struct {
Method string `json:"method"`
ExpiresIn int64 `json:"expires_in"`
}
// DownloadSet is one HTTP round-trip for up to nine Presigned GETs.
type DownloadSet struct {
Items []Download `json:"items"`
}

30
internal/module/upload/logic/credential_test.go
View File

@ -101,6 +101,36 @@ func TestCreateDownloadValidatesKey(t *testing.T) {
}
}
func TestCreateDownloadsBatch(t *testing.T) {
svc := testUpload(t)
ctx := reqctx.With(t.Context(), &reqctx.RequestContext{UserID: 1, Language: "zh-CN"})
keys := []string{
"perm/moment/image/20260918/a.jpg",
"perm/moment/image/20260918/b.jpg",
"chat/image/20260918/c.jpg",
}
got, err := svc.CreateDownloads(ctx, 1, &BatchDownloadInput{ObjectKeys: keys})
if err != nil || got == nil || len(got.Items) != 3 {
t.Fatalf("%+v %v", got, err)
}
if got.Items[0].ObjectKey != keys[0] || got.Items[1].Method != "GET" || !strings.Contains(got.Items[2].DownloadURL, keys[2]) {
t.Fatalf("%+v", got.Items)
}
if _, err := svc.CreateDownloads(ctx, 1, &BatchDownloadInput{}); err == nil {
t.Fatal("empty")
}
tooMany := make([]string, 10)
for i := range tooMany {
tooMany[i] = "chat/image/20260918/x.jpg"
}
if _, err := svc.CreateDownloads(ctx, 1, &BatchDownloadInput{ObjectKeys: tooMany}); err == nil {
t.Fatal("too many")
}
if _, err := svc.CreateDownloads(ctx, 1, &BatchDownloadInput{ObjectKeys: []string{"../secret"}}); err == nil {
t.Fatal("path escape")
}
}
func TestCreateCredentialsBatch(t *testing.T) {
svc := testUpload(t)
ctx := reqctx.With(t.Context(), &reqctx.RequestContext{UserID: 42, Language: "zh-CN"})

16
internal/module/upload/logic/download.go
View File

@ -5,7 +5,6 @@ import (
"spark-server/internal/module/upload/entity"
"spark-server/internal/pkg/errs"
"spark-server/internal/pkg/i18n"
"spark-server/internal/pkg/reqctx"
"spark-server/internal/pkg/validate"
)
@ -31,18 +30,5 @@ func (s *Service) CreateDownload(ctx context.Context, userID int64, in *Download
if !s.allow(userID) {
return nil, errs.ErrRateLimited
}
key := in.ObjectKey
if !entity.ValidObjectKey(key) {
return nil, errs.ErrInvalidArgument.WithMsg(i18n.T(reqctx.Lang(ctx), i18n.KeyUploadKeyInvalid, nil))
}
url, err := s.store.PresignGet(ctx, key, s.ttl)
if err != nil {
return nil, err
}
return &entity.Download{
ObjectKey: key,
DownloadURL: url,
Method: entity.MethodGET,
ExpiresIn: s.expiresIn(),
}, nil
return s.issueGet(ctx, in.ObjectKey)
}

64
internal/module/upload/logic/download_batch.go Normal file
View File

@ -0,0 +1,64 @@
package logic
import (
"context"
"spark-server/internal/module/upload/entity"
"spark-server/internal/pkg/errs"
"spark-server/internal/pkg/i18n"
"spark-server/internal/pkg/reqctx"
"spark-server/internal/pkg/validate"
)
const maxDownloadBatch = 9
// BatchDownloadInput issues up to nine Presigned GETs (moment 九宫格) in one call.
type BatchDownloadInput struct {
ObjectKeys []string `json:"object_keys" validate:"required,min=1,max=9,dive,required,max=512" label:"field.object_key" errmsg:"required:validate.upload.key_required;max:validate.upload.download_items_invalid"`
}
func (s *Service) CreateDownloads(ctx context.Context, userID int64, in *BatchDownloadInput) (*entity.DownloadSet, error) {
if !s.ready() {
return nil, errs.ErrAvailable
}
if userID <= 0 {
return nil, errs.ErrUnauthorized
}
if in == nil {
return nil, errs.ErrInvalidArgument
}
if err := validate.StructLang(reqctx.Lang(ctx), in); err != nil {
return nil, err
}
if n := len(in.ObjectKeys); n < 1 || n > maxDownloadBatch {
return nil, errs.ErrInvalidArgument
}
if !s.allow(userID) {
return nil, errs.ErrRateLimited
}
items := make([]entity.Download, 0, len(in.ObjectKeys))
for _, key := range in.ObjectKeys {
got, err := s.issueGet(ctx, key)
if err != nil {
return nil, err
}
items = append(items, *got)
}
return &entity.DownloadSet{Items: items}, nil
}
func (s *Service) issueGet(ctx context.Context, key string) (*entity.Download, error) {
if !entity.ValidObjectKey(key) {
return nil, errs.ErrInvalidArgument.WithMsg(i18n.T(reqctx.Lang(ctx), i18n.KeyUploadKeyInvalid, nil))
}
url, err := s.store.PresignGet(ctx, key, s.ttl)
if err != nil {
return nil, err
}
return &entity.Download{
ObjectKey: key,
DownloadURL: url,
Method: entity.MethodGET,
ExpiresIn: s.expiresIn(),
}, nil
}

3
internal/pkg/i18n/catalog_validate_upload.go
View File

@ -59,6 +59,9 @@ func init() {
KeyUploadItemsInvalid: {
ZhCN: "一次最多签发 3 个上传凭证", En: "At most 3 upload credentials per request", ZhTW: "一次最多簽發 3 個上傳憑證",
},
KeyUploadDownloadItemsInvalid: {
ZhCN: "一次最多签发 9 个下载凭证", En: "At most 9 download credentials per request", ZhTW: "一次最多簽發 9 個下載憑證",
},
KeyFavoriteMsgRequired: {
ZhCN: "请选择消息", En: "Please choose a message", ZhTW: "請選擇訊息",
},

33
internal/pkg/i18n/keys.go
View File

@ -81,22 +81,23 @@ const (
KeyFriendNotFriend = "friend.not_friend"
KeyFriendRequestInvalid = "friend.request_invalid"
KeyUploadPurposeRequired = "validate.upload.purpose_required"
KeyUploadPurposeInvalid = "validate.upload.purpose_invalid"
KeyUploadMIMERequired = "validate.upload.mime_required"
KeyUploadMIMEInvalid = "validate.upload.mime_invalid"
KeyUploadSizeRequired = "validate.upload.size_required"
KeyUploadSizeInvalid = "validate.upload.size_invalid"
KeyUploadSizeExceeded = "validate.upload.size_exceeded"
KeyUploadKeyRequired = "validate.upload.key_required"
KeyUploadKeyInvalid = "validate.upload.key_invalid"
KeyUploadGroupRequired = "validate.upload.group_required"
KeyUploadItemsRequired = "validate.upload.items_required"
KeyUploadItemsInvalid = "validate.upload.items_invalid"
KeyFavoriteMsgRequired = "validate.favorite.msg_required"
KeyFavoriteMsgInvalid = "validate.favorite.msg_invalid"
KeyMessageClientMsgRequired = "validate.message.client_msg_required"
KeyMessageContentRequired = "validate.message.content_required"
KeyUploadPurposeRequired = "validate.upload.purpose_required"
KeyUploadPurposeInvalid = "validate.upload.purpose_invalid"
KeyUploadMIMERequired = "validate.upload.mime_required"
KeyUploadMIMEInvalid = "validate.upload.mime_invalid"
KeyUploadSizeRequired = "validate.upload.size_required"
KeyUploadSizeInvalid = "validate.upload.size_invalid"
KeyUploadSizeExceeded = "validate.upload.size_exceeded"
KeyUploadKeyRequired = "validate.upload.key_required"
KeyUploadKeyInvalid = "validate.upload.key_invalid"
KeyUploadGroupRequired = "validate.upload.group_required"
KeyUploadItemsRequired = "validate.upload.items_required"
KeyUploadItemsInvalid = "validate.upload.items_invalid"
KeyUploadDownloadItemsInvalid = "validate.upload.download_items_invalid"
KeyFavoriteMsgRequired = "validate.favorite.msg_required"
KeyFavoriteMsgInvalid = "validate.favorite.msg_invalid"
KeyMessageClientMsgRequired = "validate.message.client_msg_required"
KeyMessageContentRequired = "validate.message.content_required"
KeyFieldGroupName = "field.group_name"
KeyFieldMemberUIDs = "field.member_uids"

59
internal/transport/http/docs/openapi.yaml
View File

@ -638,6 +638,7 @@ paths:
summary: 当前用户资料
description: |
Header：`Authorization: Bearer <token>`（必填），可选 `X-Device-ID`、`X-Language`。无 Body。
`avatar` 是 object_key，可匿名 GET `https://sparkr.wyrs365.com/spark/{avatar}`。
operationId: getMe
parameters:
- $ref: "#/components/parameters/X-Device-ID"
@ -1107,8 +1108,8 @@ paths:
description: |
客户端只传 `purpose` / `mime` / `size`，服务端生成 object_key 并签发短时 Presigned PUT。
文件直传 RustFS，不经过 Gin。
`avatar` 写入 `perm/avatar/{yyyyMMdd}/{name}.{ext}`（不过期，路径不含 uid）；
`group_avatar` 写入 `perm/group/{群号}/{name}.{ext}`（需 `group_id`，群主/管理员）；
`avatar` 写入 `perm/avatar/{yyyyMMdd}/{name}.{ext}`（不过期，路径不含 uid，匿名 GET）；
`group_avatar` 写入 `perm/group/{群号}/{name}.{ext}`（需 `group_id`，群主/管理员，匿名 GET）；
聊天按类型：`chat/image|voice|video|gif|file/{yyyyMMdd}/{name}.{ext}`（桶生命周期按前缀过期）。
朋友圈：`moment_image` / `moment_video` / `moment_cover` 写入 `perm/moment/{image|video|cover}/{yyyyMMdd}/{name}.{ext}`（不过期，删帖/换封面时删对象）。
`{name}` 为服务端 16 位 base62 随机串，不占用 Snowflake。
@ -1196,7 +1197,7 @@ paths:
post:
tags: [files]
summary: 签发对象存储下载凭证
description: 已登录用户用 object_key 换取短时 Presigned GET。key 必须是 `perm/` 或 `chat/` 前缀。
description: 已登录用户用 object_key 换取短时 Presigned GET。key 必须是 `perm/` 或 `chat/` 前缀。九宫格请用 `POST /v1/files/downloads/batch`。
operationId: createDownloadCredential
requestBody:
required: true
@ -1213,6 +1214,42 @@ paths:
application/json:
schema:
$ref: "#/components/schemas/Envelope"
/v1/files/downloads/batch:
post:
tags: [files]
summary: 批量签发下载凭证
description: |
一次返回 1～9 个 Presigned GET，顺序与 `object_keys` 相同。朋友圈九宫格一次拿齐。
限流按请求计入，不按条数。文件仍直传 RustFS GET。
operationId: createDownloadCredentials
requestBody:
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/CreateDownloadBatchRequest"
example:
object_keys:
- perm/moment/image/20260918/a.jpg
- perm/moment/image/20260918/b.jpg
responses:
"200":
description: 统一 envelope
content:
application/json:
schema:
$ref: "#/components/schemas/Envelope"
example:
code: "0"
msg: ok
data:
items:
- object_key: perm/moment/image/20260918/a.jpg
download_url: https://sparkr.wyrs365.com/spark/perm/moment/image/20260918/a.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256
method: GET
expires_in: 900
- object_key: perm/moment/image/20260918/b.jpg
method: GET
/v1/favorites:
get:
tags: [favorites]
@ -1296,6 +1333,7 @@ paths:
summary: 发朋友圈
description: |
纯文，或最多 9 张图，或 1 条视频（图/视频互斥）。媒体先 `POST /v1/files`（`moment_image` / `moment_video`）再填 key。
浏览九宫格：`POST /v1/files/downloads/batch` 一次换 9 张 Presigned GET。
`visibility`：0 好友公开 1 私密 2 白名单 3 黑名单。2/3 需 `audience_uids`（好友，最多 200）。
operationId: publishMoment
requestBody:
@ -2108,6 +2146,10 @@ components:
avatar:
type: string
maxLength: 512
description: |
对象存储 key。仅 `perm/avatar/`、`perm/group/` 可匿名 GET（桶本身不公开）。
URL：`{public_endpoint}/spark/{avatar}`，Nginx 也可省略桶名 `{public_endpoint}/{avatar}`。
`chat/`、`perm/fav/`、`perm/moment/` 必须 Presigned GET。
gender:
$ref: "#/components/schemas/Gender"
birthday:
@ -2211,6 +2253,17 @@ components:
object_key:
type: string
maxLength: 512
CreateDownloadBatchRequest:
type: object
required: [object_keys]
properties:
object_keys:
type: array
minItems: 1
maxItems: 9
items:
type: string
maxLength: 512
SendMessageRequest:
type: object
required: [client_msg_id, content]

13
internal/transport/http/file_test.go
View File

@ -90,6 +90,19 @@ func TestUploadCredentialHTTP(t *testing.T) {
if err := json.Unmarshal(batch.Data, &set); err != nil || len(set.Items) != 2 || set.Items[0].ObjectKey == set.Items[1].ObjectKey {
t.Fatalf("%s %v", batch.Data, err)
}
dl := authJSON(t, h, &tok, http.MethodPost, "/v1/files/downloads/batch", `{"object_keys":["chat/image/20260917/a.jpg","perm/moment/image/20260917/b.jpg"]}`)
if dl.Code != "0" {
t.Fatalf("downloads %+v", dl)
}
var dls struct {
Items []struct {
ObjectKey string `json:"object_key"`
DownloadURL string `json:"download_url"`
} `json:"items"`
}
if err := json.Unmarshal(dl.Data, &dls); err != nil || len(dls.Items) != 2 || dls.Items[0].DownloadURL == "" {
t.Fatalf("%s %v", dl.Data, err)
}
}
func TestFavoriteTextHTTP(t *testing.T) {

18
internal/transport/http/handler/file/handler.go
View File

@ -72,6 +72,24 @@ func (h *Handler) Download(c *gin.Context) {
respond.JSON(c, out)
}
func (h *Handler) DownloadBatch(c *gin.Context) {
uid, ok := currentUID(c)
if !ok {
return
}
var in uploadlogic.BatchDownloadInput
if err := respond.BindJSON(c, &in); err != nil {
respond.JSONError(c, err)
return
}
out, err := h.files.CreateDownloads(c.Request.Context(), uid, &in)
if err != nil {
respond.JSONError(c, err)
return
}
respond.JSON(c, out)
}
func currentUID(c *gin.Context) (int64, bool) {
rc := reqctx.From(c.Request.Context())
if rc == nil || rc.UserID <= 0 {

1
internal/transport/http/router.go
View File

@ -136,6 +136,7 @@ func registerFileRoutes(v1 *gin.RouterGroup, deps *EngineDeps) {
g.POST("", h.Create)
g.POST("/batch", h.CreateBatch)
g.POST("/downloads", h.Download)
g.POST("/downloads/batch", h.DownloadBatch)
}
func registerFavoriteRoutes(v1 *gin.RouterGroup, deps *EngineDeps) {