commit 6ce89511faa6290f8eba468a93a0a0f677f5a781 Author: zhr Date: Wed Sep 16 19:08:06 2026 +0800 feat: init diff --git a/.cursor/rules/go-conventions.mdc b/.cursor/rules/go-conventions.mdc new file mode 100644 index 0000000..f258351 --- /dev/null +++ b/.cursor/rules/go-conventions.mdc @@ -0,0 +1,32 @@ +--- +description: Go style matches .golangci.yaml (funlen, lll, revive, gocritic). +globs: "**/*.go" +alwaysApply: false +--- + +# Go conventions + +Lint source of truth is `.golangci.yaml`. + +- `funlen`: 100 lines / 100 statements. Split EventLoop vs business vs worker. +- `lll`: 240 columns. +- `errcheck` / `govet` / `staticcheck` / `unused` must stay clean. +- `gocritic` performance tag is on; avoid copying large structs in range. +- `revive`: real error strings, context as first argument, no unused params. +- Format with `goimports`; local prefix `spark-server`. +- Hot path: pass `context.Context`, never `context.TODO()`. +- Domain errors live in `internal/pkg/errs` and are mapped at Transport. + +```go +// BAD: EventLoop does I/O +func (s *Server) OnTraffic(c gnet.Conn) gnet.Action { + db.Create(msg) // forbidden + return gnet.None +} + +// GOOD +func (s *Server) OnTraffic(c gnet.Conn) gnet.Action { + _ = s.pool.Submit(ctx, func() { s.handleBusiness(frame) }) + return gnet.None +} +``` diff --git a/.cursor/rules/message-deliver.mdc b/.cursor/rules/message-deliver.mdc new file mode 100644 index 0000000..a791db3 --- /dev/null +++ b/.cursor/rules/message-deliver.mdc @@ -0,0 +1,19 @@ +--- +description: Message persist, deliver queue, and Redis inbox rules. +globs: internal/module/{message,deliver,inbox,sync,conversation}/**/*.go +alwaysApply: false +--- + +# Message / deliver / inbox + +Send path: permission → INSERT `messages` + UPDATE `conversations.last_msg_id` → `DeliverQueue.Push` → `SEND_ACK`. No ACK if enqueue fails. + +Idempotency: `UNIQUE(sender_id, conversation_id, client_msg_id)`. Conflicts return the original `MsgId`. + +Queue: `RPUSH` / `BRPOPLPUSH` to `deliver:processing` / `LREM` ack. On boot `RequeueStale`. Dedup with `SETNX deliver:done:{msgId}`. + +`DeliverJob` flags: `ExcludeDevice`, `NotPush`, `NotMailbox`. Sender device does not get `MESSAGE`; other devices of the sender must. + +Inbox: Redis ZSET `im:dialog:unread|read:{uid}`, preview, unread hash. Mailbox MySQL is only missed push + control events (recall/kick/friend). Never write `NEW_MESSAGE` mailbox for a successful online push. + +P2P conversation key is `(user_small, user_big)`. Sync messages with `id > last_msg_id`. diff --git a/.cursor/rules/spark-im-v3.mdc b/.cursor/rules/spark-im-v3.mdc new file mode 100644 index 0000000..0de34cb --- /dev/null +++ b/.cursor/rules/spark-im-v3.mdc @@ -0,0 +1,18 @@ +--- +description: Spark IM V3 architecture invariants. Apply to every change in spark-server. +alwaysApply: true +--- + +# Spark IM V3 invariants + +Source: `docs/architecture-v3.md`. Do not re-open these decisions. + +- C1: Message body is MySQL. Snowflake `MsgId` is PK, sort key, and sync cursor. No `ConversationSeq`. +- C2: gnet EventLoop must not call MySQL, Redis, object storage, or fanout. Inbound work goes to `TaskPool`. Fanout goes to Redis `deliver:queue` + worker. +- C3: Confidentiality is TLS. Thin Frame + optional compress only. No ECDH/AES handshake. +- C4: Public port is 443 via Nginx SNI. gnet/Gin bind loopback only. +- C5: `SEND_ACK` only after MySQL COMMIT **and** deliver `Push`. Redis enqueue failure → no ACK. +- C6: Rate limit / audit use PROXY `RealIP`, never gnet `RemoteAddr` (that is 127.0.0.1). +- C7: Conversation list/unread/preview live in Redis ZSET. Do not upsert MySQL inbox rows on the group hot path. + +Forbidden: Kafka, K8s, microservices, Keyed Worker, `context.TODO()` on hot path, kick-on-push-timeout, `im:user:{uid}:node`, business `DELIVERY_ACK`, media bytes in `messages.content`. diff --git a/.cursor/rules/tcp-eventloop.mdc b/.cursor/rules/tcp-eventloop.mdc new file mode 100644 index 0000000..46471f1 --- /dev/null +++ b/.cursor/rules/tcp-eventloop.mdc @@ -0,0 +1,21 @@ +--- +description: gnet EventLoop rules for the TCP transport. +globs: internal/transport/tcp/**/*.go +alwaysApply: false +--- + +# TCP / gnet + +EventLoop may: parse PROXY (before TLS), TLS, decode Frame, heartbeat, PUSH_ACK, rate-limit check, `TaskPool.Submit`, `AsyncWrite`. + +EventLoop must not: MySQL, Redis, upload, group member expand, deliver fanout. + +Heartbeat / PUSH_ACK stay in the loop. `AUTH` / `SEND_MESSAGE` / `SYNC_*` / `READ` / `RECALL` go to TaskPool. + +`TaskPool` is a bounded channel. Full → `errs.ErrBusy`, client retries with the same `client_msg_id`. Do not block `OnTraffic` on MySQL. + +Business modules send via `session.Manager`, never `gnet.Conn`. + +Same `device_id` kicks the old connection. Do not close a connection solely because PUSH_ACK timed out. + +gnet options: Multicore, 16MB buffers, TCPKeepAlive 30s. IdleTimeout 120s. Align with Nginx `proxy_timeout`. diff --git a/.cursor/skills/spark-im-v3/SKILL.md b/.cursor/skills/spark-im-v3/SKILL.md new file mode 100644 index 0000000..e6e4003 --- /dev/null +++ b/.cursor/skills/spark-im-v3/SKILL.md @@ -0,0 +1,63 @@ +--- +name: spark-im-v3 +description: >- + Implements Spark IM modular monolith per architecture V3. Use when writing or + reviewing spark-server Go code, TCP/gnet, Gin HTTP, messages, deliver queue, + inbox ZSET, Nginx 443 SNI, PROXY protocol, or SEND_ACK. Trigger terms: IM, + MsgId, TaskPool, deliver:queue, SYNC_INBOX, gnet, RealIP. +--- + +# Spark IM V3 + +Read [invariants.md](invariants.md) before changing send/sync/transport. Full design: `docs/architecture-v3.md`. Lint: `.golangci.yaml`. + +## Layout + +```text +cmd/server process entry +internal/bootstrap config, lifecycle +internal/transport/http Gin, loopback :8080 +internal/transport/tcp gnet EventLoop + TaskPool + Frame +internal/module/* Application services (HTTP and TCP share these) +internal/infrastructure mysql, redis, id, logger, push +``` + +Module files: `service.go` + entity types + `repository.go` interface. No extra DDD layer. + +## Send path (must keep this order) + +1. TaskPool (not EventLoop) +2. Permission (friend / block / group mute) — fail before INSERT +3. Snowflake MsgId (or client MsgId within ±5m) +4. MySQL txn: `messages` + `conversations.last_msg_id` +5. `DeliverQueue.Push` (reliable list) +6. `SEND_ACK` + +If step 5 fails, do not ACK. Client retries the same `client_msg_id`. + +## EventLoop vs pools + +| Pool | Owns | +|------|------| +| gnet EventLoop | PROXY, TLS, Frame, heartbeat, PUSH_ACK, Submit | +| TaskPool | AUTH, SEND, SYNC, READ, RECALL, MySQL | +| Deliver Worker | ZSET inbox, Session.Send, mailbox on push fail, PushNotifier | + +## Commands + +`AUTH HEARTBEAT GENERATE_MSG_ID SEND_MESSAGE SEND_ACK MESSAGE PUSH_ACK READ RECALL SYNC_INBOX SYNC_MESSAGES KICK` + +Media: HTTP upload credential, message content is object key only. Frame max 64KB. Compress body over 512B. + +## When adding a feature + +- HTTP writes must enqueue deliver jobs for online devices. +- List/unread go to Redis ZSET, not MySQL row-per-member. +- New goroutine I/O does not belong in `OnTraffic`. +- Run `make lint` (funlen 100). + +## Additional resources + +- [invariants.md](invariants.md) +- [send-path.md](send-path.md) +- `docs/architecture-v3.md` diff --git a/.cursor/skills/spark-im-v3/invariants.md b/.cursor/skills/spark-im-v3/invariants.md new file mode 100644 index 0000000..9006d58 --- /dev/null +++ b/.cursor/skills/spark-im-v3/invariants.md @@ -0,0 +1,13 @@ +# V3 invariants (short) + +| ID | Rule | +|----|------| +| C1 | MySQL + Snowflake MsgId. No ConversationSeq. | +| C2 | EventLoop ≠ MySQL. Fanout = Redis list + worker (`BRPOPLPUSH`). | +| C3 | TLS + thin Frame. No custom handshake crypto. | +| C4 | Public 443 / Nginx SNI. gnet is `127.0.0.1:9000`. | +| C5 | ACK = COMMIT + enqueue. | +| C6 | PROXY RealIP for limit/audit. | +| C7 | Dialog list in Redis ZSET. Mailbox only for missed push + control events. | + +P2P unique `(user_small, user_big)`. Same device kicks old conn. Sender device excluded from MESSAGE. `context.TODO()` banned on hot path. Push timeout does not close TCP. diff --git a/.cursor/skills/spark-im-v3/send-path.md b/.cursor/skills/spark-im-v3/send-path.md new file mode 100644 index 0000000..3e360cb --- /dev/null +++ b/.cursor/skills/spark-im-v3/send-path.md @@ -0,0 +1,33 @@ +# Send / sync examples + +## Persist then enqueue then ACK + +```go +msg.ID, err = idgen.NextMsgID() +if err != nil { return nil, err } +if err := repo.Create(ctx, msg); err != nil { return nil, err } +if err := convRepo.UpdateLastMsg(ctx, msg.ConversationID, msg.ID, now); err != nil { return nil, err } +if err := deliver.Push(ctx, &deliver.Job{MsgID: msg.ID, ExcludeDevice: req.DeviceID}); err != nil { + return nil, err // no SEND_ACK +} +return &SendResponse{MsgID: msg.ID, ClientMsgID: req.ClientMsgID, ServerTimeMS: now}, nil +``` + +## EventLoop submit + +```go +if frame.Command == protocol.CmdHeartbeat { + return writeHeartbeat(c, frame) +} +_ = pool.Submit(ctx, func() { router.Dispatch(ctx, frame) }) +``` + +## Sync + +```sql +SELECT * FROM messages + WHERE conversation_id = ? AND id > ? + ORDER BY id ASC LIMIT 200; +``` + +Inbox sync reads Redis ZSET by `dialog_score`, then mailbox events by `mailbox_seq`. diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..713b24b --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +bin/ +vendor/ +*.exe +*.test +*.out +.idea/ +.vscode/ +*.log +coverage.out +.env +certs/*.key diff --git a/.golangci.yaml b/.golangci.yaml new file mode 100644 index 0000000..47f9275 --- /dev/null +++ b/.golangci.yaml @@ -0,0 +1,92 @@ +# Spark IM coding conventions for golangci-lint v1. +# Ported from BFF .golangci.yml (funlen/gocritic/lll/enable set) for Go 1.26. +run: + timeout: 10m + tests: true + go: "1.26" + +linters-settings: + funlen: + lines: 100 + statements: 100 + lll: + line-length: 240 + misspell: + locale: US + gocritic: + enabled-tags: + - performance + settings: + captLocal: + paramsOnly: true + rangeValCopy: + sizeThreshold: 32 + goconst: + min-len: 3 + min-occurrences: 3 + dupl: + threshold: 150 + goimports: + local-prefixes: spark-server + revive: + rules: + - name: blank-imports + - name: context-as-argument + - name: context-keys-type + - name: dot-imports + - name: error-return + - name: error-strings + - name: error-naming + - name: exported + - name: if-return + - name: increment-decrement + - name: var-naming + - name: var-declaration + - name: package-comments + - name: range + - name: receiver-naming + - name: time-naming + - name: unexported-return + - name: indent-error-flow + - name: errorf + - name: empty-block + - name: superfluous-else + - name: unused-parameter + - name: unreachable-code + - name: redefines-builtin-id + +linters: + enable: + - errcheck + - gosimple + - govet + - ineffassign + - staticcheck + - unused + - misspell + - dupl + - unparam + - gocritic + - funlen + - goconst + - unconvert + - bodyclose + - sqlclosecheck + - gomodguard + - revive + - lll + - goimports + +issues: + exclude: + - "SA1029" + - "SA1019" + exclude-rules: + - path: _test\.go + linters: + - dupl + - funlen + - goconst + - path: internal/module/.*/entity\.go + linters: + - unused diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..a33e3b4 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,12 @@ +# spark-server agent notes + +This repo is a Go 1.26 modular monolith IM server. Source of truth: `docs/architecture-v3.md`. + +Before writing code: + +1. Load `.cursor/skills/spark-im-v3/SKILL.md` +2. Follow `.cursor/rules/` (always-on V3 invariants + file-scoped rules) +3. Keep functions within `.golangci.yaml` (`funlen` 100, `lll` 240) +4. HTTP and TCP share Application services under `internal/module/*` + +Do not introduce Kafka, K8s, microservices, ConversationSeq, application-layer handshake crypto, or public gnet ports. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a47d11d --- /dev/null +++ b/Makefile @@ -0,0 +1,29 @@ +GO ?= go +MODULE := spark-server +BIN := bin/spark-server +LINT ?= golangci-lint + +.PHONY: all build test lint fmt vet tidy run + +all: lint test build + +build: + $(GO) build -o $(BIN) ./cmd/server + +test: + $(GO) test ./... + +vet: + $(GO) vet ./... + +fmt: + $(GO) fmt ./... + +lint: + $(LINT) run ./... + +tidy: + $(GO) mod tidy + +run: build + ./$(BIN) -config configs/config.yaml diff --git a/README.md b/README.md new file mode 100644 index 0000000..dc655f8 --- /dev/null +++ b/README.md @@ -0,0 +1,16 @@ +# spark-server + +Spark IM modular monolith. Architecture: [docs/architecture-v3.md](docs/architecture-v3.md). + +- Go 1.26 +- Gin HTTP on `127.0.0.1:8080` +- gnet TCP on `127.0.0.1:9000` (EventLoop + TaskPool). gnet v2.10 has no built-in TLS; production terminates TLS at Nginx or a PROXY+TLS wrapper in front of gnet. Loopback is not public. +- Public entry is Nginx `:443` only (SNI split) + +```bash +go build -o bin/spark-server ./cmd/server +./bin/spark-server -config configs/config.yaml +make lint +``` + +Agent conventions live in `.cursor/rules` and `.cursor/skills/spark-im-v3`. Lint conventions live in `.golangci.yaml`. diff --git a/api/proto/common.proto b/api/proto/common.proto new file mode 100644 index 0000000..f393a8b --- /dev/null +++ b/api/proto/common.proto @@ -0,0 +1,23 @@ +syntax = "proto3"; + +package spark.im.v1; + +option go_package = "spark-server/api/gen/imv1"; + +enum Command { + COMMAND_UNSPECIFIED = 0; + AUTH = 1; + HEARTBEAT = 2; + GENERATE_MSG_ID = 3; + SEND_MESSAGE = 4; + SEND_ACK = 5; + MESSAGE = 6; + PUSH_ACK = 7; + READ_MESSAGE = 8; + RECALL_MESSAGE = 9; + SYNC_INBOX = 10; + SYNC_INBOX_FINISH = 11; + SYNC_MESSAGES = 12; + SYNC_MESSAGES_FINISH = 13; + KICK = 14; +} diff --git a/api/proto/message.proto b/api/proto/message.proto new file mode 100644 index 0000000..fad54ec --- /dev/null +++ b/api/proto/message.proto @@ -0,0 +1,31 @@ +syntax = "proto3"; + +package spark.im.v1; + +option go_package = "spark-server/api/gen/imv1"; + +message SendMessageRequest { + int64 conversation_id = 1; + string client_msg_id = 2; + int32 message_type = 3; + bytes content = 4; + int64 msg_id = 5; +} + +message SendMessageResponse { + int64 msg_id = 1; + string client_msg_id = 2; + int64 server_time = 3; +} + +message SyncMessagesRequest { + int64 conversation_id = 1; + int64 last_msg_id = 2; + int32 limit = 3; +} + +message SyncInboxRequest { + uint64 device_id = 1; + int64 mailbox_seq = 2; + int64 dialog_score = 3; +} diff --git a/cmd/server/main.go b/cmd/server/main.go new file mode 100644 index 0000000..d3220d7 --- /dev/null +++ b/cmd/server/main.go @@ -0,0 +1,26 @@ +package main + +import ( + "flag" + "log" + "os" + + "spark-server/internal/bootstrap" +) + +func main() { + cfgPath := flag.String("config", "configs/config.yaml", "config path") + flag.Parse() + cfg, err := bootstrap.Load(*cfgPath) + if err != nil { + log.Fatal(err) + } + app, err := bootstrap.New(cfg) + if err != nil { + log.Fatal(err) + } + if err := app.Run(); err != nil { + log.Fatal(err) + } + os.Exit(0) +} diff --git a/configs/config.yaml b/configs/config.yaml new file mode 100644 index 0000000..66e4730 --- /dev/null +++ b/configs/config.yaml @@ -0,0 +1,51 @@ +server: + http: + addr: "127.0.0.1:8080" + tcp: + addr: "127.0.0.1:9000" + proxy_protocol: true + tls: + cert_file: "" + key_file: "" + max_frame_size: 65536 + idle_timeout: 120s + keepalive: 30s + recv_buffer: 16777216 + send_buffer: 16777216 + multicore: true + compress_min_size: 512 + +id: + snowflake_worker_id: 1 + +mysql: + dsn: "" + +redis: + addr: "" + deliver_queue_key: "deliver:queue" + deliver_processing_key: "deliver:processing" + +taskpool: + size: 256 + queue: 1024 + +deliver: + workers: 8 + take_timeout: 5s + done_ttl: 10m + +session: + outbound_queue_size: 1024 + +rate_limit: + message_per_second: 20 + auth_per_second: 5 + sync_inbox_per_minute: 60 + sync_messages_per_minute: 120 + history_per_minute: 200 + +message: + max_content_size: 65536 + sync_limit: 200 + msgid_expire_ms: 300000 diff --git a/deploy/nginx/http.conf b/deploy/nginx/http.conf new file mode 100644 index 0000000..39a696f --- /dev/null +++ b/deploy/nginx/http.conf @@ -0,0 +1,14 @@ +http { + server { + listen 127.0.0.1:8443 ssl; + server_name api.example.com; + + location / { + proxy_pass http://127.0.0.1:8080; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + } + } +} diff --git a/deploy/nginx/stream.conf b/deploy/nginx/stream.conf new file mode 100644 index 0000000..700120c --- /dev/null +++ b/deploy/nginx/stream.conf @@ -0,0 +1,24 @@ +stream { + map $ssl_preread_server_name $backend { + api.example.com 127.0.0.1:8443; + im.example.com 127.0.0.1:9001; + default ""; + } + + server { + listen 443 reuseport; + ssl_preread on; + proxy_connect_timeout 5s; + proxy_timeout 3600s; + proxy_pass $backend; + } + + # Add PROXY protocol only on the IM hop so Gin/HTTP is unaffected. + server { + listen 127.0.0.1:9001; + proxy_protocol on; + proxy_connect_timeout 5s; + proxy_timeout 3600s; + proxy_pass 127.0.0.1:9000; + } +} diff --git a/docs/architecture-v3.md b/docs/architecture-v3.md new file mode 100644 index 0000000..73811cc --- /dev/null +++ b/docs/architecture-v3.md @@ -0,0 +1,890 @@ +# 生产版 Go 单体 IM 系统架构设计 V3.0 + +> 版本：V3.0 +> 基于：V2.0 + 对照 BFF 现网补上的可靠性、入口与会话列表切分 +> 架构形态：Go Modular Monolith（模块化单体） +> 适用阶段：单节点 / 单地域生产环境 +> 核心原则：**先保证正确性，再根据压测结果引入复杂度** + +不上 Kafka、K8s、微服务、Keyed Worker。机器一台、公网 IP 一个、公网端口只有 443。 + +--- + +# 0. 硬约束（继承 V2，V3 再钉三条） + +| # | 约束 | 含义 | +| --- | --- | --- | +| C1 | 消息存储 = MySQL + MsgId | 正文只在 MySQL。Snowflake `MsgId` 是主键、会话内排序和增量游标。没有 ConversationSeq。 | +| C2 | EventLoop 与 MySQL 拆开；扇出走 Redis list | 学 BFF `deliversv`。入队用可靠队列，不是裸 `BLPOP`。 | +| C3 | gnet 不自研加密协议，走 TLS | TLS 负责保密。应用层只有薄 Frame + 可选压缩。 | +| C4 | 公网只开 443；Nginx 按 SNI 分流 | HTTP / IM 两个域名。gnet 只绑回环。 | +| C5 | `SEND_ACK` = 已落库 **且** 已入扇出队列 | `COMMIT → 入队 → ACK`。Redis 失败不 ACK。 | +| C6 | 透传必须带真实 IP | Nginx `proxy_protocol`；gnet 在 TLS 之前读 PROXY 头。 | +| C7 | 会话列表 / 未读在 Redis | 学 BFF ZSET。MySQL 不承担群消息 N 行 inbox 更新。 | + +相对 V2 明确撤回： + +* 撤回 `COMMIT → SEND_ACK → RPUSH`（中间崩溃会对端无感知） +* 撤回 MySQL `user_inbox` 作为热路径会话列表 +* 撤回小群给每个成员写 `NEW_MESSAGE` inbox_event +* 撤回裸 `BLPOP` 作为唯一消费方式 +* 撤回「gnet 用 RemoteAddr 当客户端 IP」（那是 127.0.0.1） + +不抄 BFF：自定义 ECDH/AES、Push 超时拆连接、`SequenceID > 200` 才限流、CheckToken 绑 Redis conn、`context.TODO()` 热路径、`netstat` 数在线。 + +--- + +# 1. 总览 + +```text + Internet + │ + :443 TLS + │ + Nginx stream + SNI + + proxy_protocol + ┌─────────────┴─────────────┐ + │ api.example.com │ im.example.com + ▼ ▼ + 127.0.0.1:8443 127.0.0.1:9000 + Nginx 终结 TLS gnet：PROXY → TLS + │ │ + 127.0.0.1:8080 EventLoop + Gin │ + │ TaskPool + └──────────┬───────────────┘ + ▼ + Application + │ + ┌────────────┼────────────┐ + ▼ ▼ ▼ + MySQL Redis Session + 消息正文 列表/未读 本机连接 + 成员/已读 预览/限流 + 控制邮箱 deliver 队列 + │ + Deliver Worker + 列表 + Push + 失败邮箱 +``` + +> **实时靠连接，正文靠 MySQL，列表靠 Redis，扇出靠可靠队列，对外只露 443。** + +`SEND_ACK` 表示：消息已在 MySQL，且 deliver 任务已在 Redis 队列里。不表示对方已收到。对方最终可见靠：在线 Push，或上线后 `SYNC_INBOX` + `SYNC_MESSAGES`。 + +--- + +# 2. 部署与入口 + +## 2.1 监听 + +```text +公网 :443 +Gin 127.0.0.1:8080 +本机 HTTPS 127.0.0.1:8443 Nginx 终结 api 证书 +gnet 127.0.0.1:9000 PROXY 头 + 终结 im 证书 +``` + +```text +HTTPS API https://api.example.com +IM 长连接 im.example.com:443 +``` + +## 2.2 Nginx：SNI + PROXY protocol + +IM 是自定义 TCP，不是 HTTP。stream 用 `ssl_preread` 读 SNI，对 IM **透传 TLS**，并在转发给 gnet 时加上 PROXY protocol，否则 gnet 只看得到 127.0.0.1。 + +```nginx +stream { + map $ssl_preread_server_name $backend { + api.example.com 127.0.0.1:8443; + im.example.com 127.0.0.1:9000; + default ""; + } + + map $ssl_preread_server_name $proxy_proto { + im.example.com 1; + default 0; + } + + server { + listen 443 reuseport; + ssl_preread on; + proxy_connect_timeout 5s; + proxy_timeout 3600s; + proxy_protocol on; # 仅对需要的 upstream 生效见下 + proxy_pass $backend; + } +} +``` + +未知 SNI **不要**落到 HTTP。`default` 为空则拒绝连接。IM 证书必须匹配 `im.example.com`。 + +HTTP： + +```nginx +http { + server { + listen 127.0.0.1:8443 ssl; + server_name api.example.com; + location / { + proxy_pass http://127.0.0.1:8080; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + } + } +} +``` + +若 stream 全局 `proxy_protocol on` 会影响 8443，则拆成两个 `server` / 两个 `map`：只对 `im.example.com` 打开 `proxy_protocol`。以本机实测为准，原则是：**只有 gnet 收到 PROXY 头，Gin 仍走 HTTP 头里的真实 IP。** + +## 2.3 gnet：先 PROXY，再 TLS + +```text +accept + → 读 PROXY v1/v2（超时要短，非法头断开） + → 取出 src IP/port 写入 ConnInfo.RealIP + → TLS handshake（证书 = im.example.com） + → Frame +``` + +限流、封禁、审计、日志一律用 `RealIP`，禁止用 `127.0.0.1`。 + +## 2.4 超时必须对齐 + +| 项 | 值 | 理由 | +| --- | --- | --- | +| 客户端心跳 | 30s | 与 BFF `TCPKeepAlive` 同量级 | +| gnet `TCPKeepAlive` | 30s | 维持 Nginx 流 | +| 连接 `IdleTimeout` | 120s | 连续错过心跳才断 | +| Nginx `proxy_timeout` | 3600s | 长连接；靠 keepalive 探活 | +| 本机探活 | Gin `/healthz`；gnet 管理口或 TCP 本机 | 公网安全组只放 443 | + +三条对不齐时，客户端会表现为「IM 随机断线」，实际是 Nginx 静默掐流。BFF 没有这一跳，所以 V3 必须把对齐写进验收。 + +## 2.5 gnet Options（搬 connectorsv） + +```text +Multicore true +ReusePort true +SocketRecvBuffer 16MB +SocketSendBuffer 16MB +TCPKeepAlive 30s +``` + +--- + +# 3. 为什么仍是单体 + +```text +443 → gnet → TaskPool → MySQL → 入队 → ACK + Redis list → Deliver Worker → 列表 / Push +``` + +HTTP 与 IM 共用 Application。拆微服务只会在这一跳加 RPC。 + +--- + +# 4. 目录 + +```text +im-server/ +├── cmd/server/main.go +├── internal/ +│ ├── bootstrap/ +│ ├── transport/ +│ │ ├── http/ +│ │ └── tcp/ +│ │ ├── server.go # PROXY + TLS +│ │ ├── connection.go +│ │ ├── command_router.go +│ │ ├── taskpool.go +│ │ ├── protocol/ # 薄 Frame + 压缩 +│ │ └── session/ +│ ├── module/ +│ │ ├── auth/ +│ │ ├── user/ +│ │ ├── friend/ +│ │ ├── group/ +│ │ ├── conversation/ +│ │ ├── message/ +│ │ ├── inbox/ # Redis ZSET 列表 + 失败邮箱 +│ │ ├── sync/ +│ │ ├── session/ +│ │ ├── upload/ +│ │ └── deliver/ +│ └── infrastructure/ +│ ├── mysql/ +│ ├── redis/ +│ ├── id/ # Snowflake +│ ├── logger/ +│ ├── metrics/ +│ ├── tracing/ +│ └── push/ +├── deploy/nginx/ +├── api/proto/ +├── configs/ +└── migrations/ +``` + +模块内：`service.go` + `entity.go` + `repository.go`。不先堆 DDD。 + +--- + +# 5. 分层 + +```text +Nginx 443 + → Transport（Gin / gnet） + → TaskPool // inbound，含 MySQL + → Application + → MySQL | Redis | ObjectStore | PushNotifier +``` + +禁止：EventLoop 跑 MySQL / 扇出；`MessageService` 碰 `gnet.Conn`；发送路径同步推完所有设备。 + +--- + +# 6. Transport + +## 6.1 EventLoop 硬规则 + +```text +允许：PROXY、TLS、拆包、心跳、限流判定、Submit TaskPool、AsyncWrite +禁止：MySQL、Redis 同步调用、对象存储、群成员展开、Deliver 扇出 +``` + +两套池： + +```text +TaskPool 发送方延迟（AUTH / SEND / SYNC / READ / RECALL） +Deliver Worker 扇出延迟（列表、Push、失败邮箱） +``` + +`SEND_ACK` 只等 TaskPool 里的 MySQL + 入队。 + +## 6.2 TaskPool + +```go +type TaskPool interface { + Submit(ctx context.Context, task func()) error +} +``` + +有界 channel 即可。满则 `ErrBusy`，客户端用同一 `client_msg_id` 重试。心跳 / PROXY / PUSH_ACK 留在 EventLoop。 + +热路径必须带可取消的 `context.Context`，禁止 `context.TODO()`。 + +--- + +# 7. 协议：TLS + 薄 Frame + +不学 BFF 握手加密、双版本头、把 Response 当 Request。 + +```text +TCP → PROXY → TLS 1.2+ → Frame → Protobuf +``` + +```text +Magic u16 | Version u8 | Command u16 | Flags u16 +RequestID u64 | SequenceID u32 | BodyLength u32 +Protobuf Body +``` + +`Flags` 含 `Compress`。Body 超过 512B 再压缩（阈值可配），对齐 BFF `CompressMinSize`。 + +`SequenceID` 只匹配请求/响应和 PUSH_ACK。 +`MaxFrameSize = 64KB`。超大 `BodyLength` 断开。媒体不走 Frame。 + +## 7.1 Command + +TCP： + +```text +AUTH HEARTBEAT +GENERATE_MSG_ID +SEND_MESSAGE SEND_ACK MESSAGE PUSH_ACK +READ_MESSAGE RECALL_MESSAGE +SYNC_INBOX SYNC_INBOX_FINISH +SYNC_MESSAGES SYNC_MESSAGES_FINISH +KICK +``` + +HTTP：资料、好友、群管理、会话列表分页、历史、上传凭证、设置。 +HTTP 写操作走同一 Application，并入 deliver 队列。 + +--- + +# 8. Session + +```go +type Connection interface { + ID() string + UserID() int64 + DeviceID() uint64 + RealIP() string + Send(packet *Packet) error + Close() error +} + +type SessionManager interface { + Bind(session Session) error + Unbind(sessionID string) error + GetUserSessions(userID int64) []Session + SendToUserExceptDevice(ctx context.Context, userID int64, exceptDeviceID uint64, packet *Packet) error +} +``` + +```text +同 device_id 踢旧连接，下发 KICK +多 device 允许 +发送设备 不回推 MESSAGE +其他设备 必须推 +cursor (user_id, device_id) +``` + +出站有界队列，满则丢实时、计 `outbound_drop`。PUSH_ACK 超时只降级到失败邮箱 / PushNotifier，**不拆连接**。 + +连接态：`CONNECTING → CONNECTED → AUTHENTICATED → ACTIVE → CLOSING → CLOSED`。AUTH 一次，身份挂 Session。 + +不写 `im:user:{uid}:node`。 + +--- + +# 9. 消息：MySQL + MsgId + +## 9.1 ID + +```text +MsgId Snowflake，主键、排序、同步游标 +ClientMsgID 客户端幂等 +TCP SequenceID 请求匹配 +InboxSeq 仅失败邮箱 / 控制事件 +``` + +Snowflake worker id 写进配置并固定到这台机器。时钟回拨：拒绝发号或等待，禁止生成回退的 MsgId。重启后 worker id 不变。 + +## 9.2 GENERATE_MSG_ID + +学 BFF `0x0219`。客户端可先拿 MsgId 做乐观 UI，再 `SEND_MESSAGE` 带同一 id。 + +若请求自带 MsgId：校验其时间戳在 ±5 分钟内；已存在则走幂等返回；非法则服务端重发一个。 + +## 9.3 发送路径（C5） + +```text +TaskPool: + 1. RealIP 限流 + 2. 权限（见 9.4），失败不写库 + 3. MsgId = 请求带来的或 Snowflake + 4. BEGIN + INSERT messages + UPDATE conversations.last_msg_id / last_msg_at + COMMIT + 5. DeliverQueue.Push // 可靠入队，失败则重试有限次 + 6. 仍失败 → 不 SEND_ACK，客户端用 client_msg_id 重试 + 7. SEND_ACK(msg_id, client_msg_id, server_time) +``` + +幂等：`UNIQUE(sender_id, conversation_id, client_msg_id)`。冲突查出原行返回原 MsgId；若原消息尚未在队列里（极端），补一次 `Push` 再 ACK。 + +## 9.4 发送前校验（事务前完成） + +```text +P2P 好友关系、未拉黑、对方未禁你 +群 是成员、未全员禁言 / 未被单独禁言 +通用 类型合法、content ≤ 64KB、媒体只允许引用 +``` + +失败不入队。系统号走单独开关，可 `NotPush`。 + +--- + +# 10. 表 + +## 10.1 messages + +```sql +CREATE TABLE messages ( + id BIGINT PRIMARY KEY, + conversation_id BIGINT NOT NULL, + sender_id BIGINT NOT NULL, + client_msg_id VARCHAR(64) NOT NULL, + message_type SMALLINT NOT NULL, + content VARBINARY(65535) NOT NULL, + status TINYINT NOT NULL DEFAULT 1, + created_at DATETIME(3) NOT NULL, + UNIQUE KEY uk_sender_conv_client_msg (sender_id, conversation_id, client_msg_id), + KEY idx_conversation_id (conversation_id, id) +); +``` + +```sql +SELECT * FROM messages + WHERE conversation_id = ? AND id > ? + ORDER BY id ASC LIMIT 200; +``` + +## 10.2 conversations / P2P 唯一 / members + +```sql +CREATE TABLE conversations ( + id BIGINT PRIMARY KEY, + type TINYINT NOT NULL, -- 1=P2P 2=GROUP + last_msg_id BIGINT NOT NULL DEFAULT 0, + last_msg_at DATETIME(3) NOT NULL, + created_at DATETIME(3) NOT NULL +); + +-- 两人永远同一条会话，学 BFF 小 uid : 大 uid +CREATE TABLE p2p_conversations ( + user_small BIGINT NOT NULL, + user_big BIGINT NOT NULL, + conversation_id BIGINT NOT NULL, + PRIMARY KEY (user_small, user_big), + UNIQUE KEY uk_conversation (conversation_id) +); + +CREATE TABLE conversation_members ( + conversation_id BIGINT NOT NULL, + user_id BIGINT NOT NULL, + last_read_msg_id BIGINT NOT NULL DEFAULT 0, + joined_at DATETIME(3) NOT NULL, + muted TINYINT NOT NULL DEFAULT 0, + PRIMARY KEY (conversation_id, user_id), + KEY idx_user_id (user_id) +); +``` + +`last_read_msg_id` 在 MySQL，作为已读真相（重启 Redis 可从这里重建未读）。 +会话列表不扫这张表的热路径。 + +不做 `message_receipts`。 + +## 10.3 mailbox_events（失败邮箱 + 控制事件） + +```sql +CREATE TABLE mailbox_events ( + id BIGINT PRIMARY KEY, + user_id BIGINT NOT NULL, + device_id BIGINT NOT NULL, -- 0 = 该用户所有设备 + seq BIGINT NOT NULL, + event_type SMALLINT NOT NULL, -- RECALL / KICK / MEMBER / FRIEND / MISSED_MESSAGE + conversation_id BIGINT NULL, + payload VARBINARY(4096), -- conversation_id + msg_id，无正文 + created_at DATETIME(3) NOT NULL, + UNIQUE KEY uk_user_device_seq (user_id, device_id, seq), + KEY idx_user_device (user_id, device_id, seq) +); + +CREATE TABLE device_sync_cursors ( + user_id BIGINT NOT NULL, + device_id BIGINT NOT NULL, + mailbox_seq BIGINT NOT NULL DEFAULT 0, + dialog_score BIGINT NOT NULL DEFAULT 0, -- Redis ZSET score 游标 + updated_at DATETIME(3) NOT NULL, + PRIMARY KEY (user_id, device_id) +); +``` + +**不**给每条成功 Push 的新消息写 mailbox。对齐 BFF ICH：邮箱只服务没推上的设备，以及撤回 / 踢人 / 好友这类控制事件。 + +--- + +# 11. 会话列表：Redis ZSET（C7） + +对齐 BFF `unread:dialog:{uid}` / `read:dialog:{uid}`。 + +```text +im:dialog:unread:{uid} ZSET score=last_msg_at_ms member=conversation_id +im:dialog:read:{uid} ZSET 同上 +im:dialog:unreadcnt:{uid} HASH field=conversation_id value=count +im:dialog:preview:{cid} STRING 最后一条短预览（id/type/text/sender） +im:dialog:muted:{uid} SET +im:dialog:pinned:{uid} ZSET +``` + +Deliver Worker 对每个收件人（pipeline）： + +```text +ZADD unread 或 read（免打扰进 read，score 仍更新） +HINCRBY unreadcnt （免打扰 / 发送者自己不加；按 MsgId 幂等） +SET preview +``` + +列表接口 / `SYNC_INBOX` 的会话差量： + +```text +ZRANGEBYSCORE ... (last_dialog_score +inf +``` + +不要每条群消息 `UPDATE` N 行 MySQL inbox。MySQL 只在：建会话、改成员、持久化 `last_read_msg_id`、控制事件落 mailbox。 + +预览与未读以 Redis 为准；Redis 丢失后： + +```text +会话列表可降级扫 conversation_members + conversations.last_msg_at（冷路径） +未读用 last_read_msg_id 与 last_msg_id 重算 +消息正文不受影响 +``` + +--- + +# 12. 扇出：可靠 Redis 队列 + +## 12.1 结构 + +```text +deliver:queue LIST 待处理 +deliver:processing LIST 已取出未完成 +deliver:done:{msgId} STRING 短 TTL，幂等 +``` + +```go +type DeliverJob struct { + MsgID int64 + Cmd uint16 + ConversationID int64 + FromUserID int64 + FromDeviceID uint64 + ExcludeDevice uint64 + NotPush bool + NotMailbox bool // 对应 BFF NotSaveIch + Payload []byte +} + +type DeliverQueue interface { + Push(ctx context.Context, job *DeliverJob) error + Take(ctx context.Context, timeout time.Duration) (*DeliverJob, error) // RPOPLPUSH + Ack(ctx context.Context, job *DeliverJob) error // LREM processing + RequeueStale(ctx context.Context) error +} +``` + +生产：`RPUSH deliver:queue`。失败则发送路径不 ACK。 + +消费：`RPOPLPUSH queue → processing`，处理完 `LREM processing`。进程崩溃：启动时把 `processing` 搬回 `queue`。这修补了 BFF 裸 `BLPOP` 丢任务的问题。 + +## 12.2 Worker + +```text +job = Take +if SETNX deliver:done:{msgId} 失败 → Ack 丢掉，防止 unread +2 +1. 收件人（P2P / 群成员缓存） +2. 按 ExcludeDevice / muted / NotPush 过滤 +3. pipeline 更新 ZSET 列表、预览、未读 +4. Session.Send 在线设备 +5. 写出失败或无连接： + NotMailbox=false → mailbox_events（该 device） + NotPush=false 且未 mute → PushNotifier +Ack(job) +``` + +群成员列表要有短缓存，不要每条消息打全表。 + +## 12.3 旗帜（搬 BFF） + +```text +ExcludeDeviceId 发送设备 +NotPush 系统消息、已读回执等不打系统推送 +NotMailbox 不需要失败邮箱（例如纯在线信令） +``` + +规则在 Worker，不在 gnet。 + +## 12.4 Redis 丢队列 + +`SEND_ACK` 之后才允许 Redis 丢数据的窗口只剩「队列里还没消费」。处理中列表用于扛 Worker 崩溃。若 Redis 整实例丢失： + +```text +正文仍在 MySQL +列表可重建 +实时 Push 可能漏 +上线 SYNC_MESSAGES 按 last_msg_id 仍能补已知会话 +新会话感知会弱一档（直到有下一次列表重建） +``` + +这是 C7 的代价，写进不变量，不把 Redis 当消息真相。 + +--- + +# 13. 两级同步 + +```text +AUTH + → SYNC_INBOX(device_id, mailbox_seq, dialog_score) + · ZSET 差量：会话列表 / 预览 / 未读 + · mailbox 差量：MISSED_MESSAGE、RECALL、KICK、好友、成员 + → 对 dirty conversation + SYNC_MESSAGES(conversation_id, last_msg_id) +``` + +限流按 `device_id` 分开（学 BFF 历史 200 次窗口）： + +```text +SYNC_INBOX +SYNC_MESSAGES +HTTP GetHistory +SEND_MESSAGE +AUTH +``` + +从第一条业务命令开始限流，不要 `SequenceID > 200`。 + +--- + +# 14. ACK + +```text +SEND_ACK 已落库且已入队 +PUSH_ACK 传输层，不落业务库 +READ 更新 last_read_msg_id，清 Redis 未读，入队通知其他设备 +``` + +不做业务 DELIVERY_ACK。 + +已读：MySQL `last_read_msg_id` + Redis `HDEL unreadcnt` + 从 unread ZSET 挪到 read ZSET。 + +--- + +# 15. 模块接口 + +```go +type MessageService interface { + GenerateMsgID(ctx context.Context, req *GenerateMsgIDRequest) (int64, error) + SendMessage(ctx context.Context, req *SendMessageRequest) (*SendMessageResponse, error) + RecallMessage(ctx context.Context, req *RecallMessageRequest) error + MarkRead(ctx context.Context, req *MarkReadRequest) error + GetHistory(ctx context.Context, req *GetHistoryRequest) (*GetHistoryResponse, error) +} + +type SyncService interface { + SyncInbox(ctx context.Context, req *SyncInboxRequest) (*SyncInboxResponse, error) + SyncMessages(ctx context.Context, req *SyncMessagesRequest) (*SyncMessagesResponse, error) +} + +type InboxStore interface { /* Redis ZSET + preview */ } +type MailboxRepository interface { /* 失败/控制事件 */ } +type DeliverQueue interface{} +type PushNotifier interface { + Notify(ctx context.Context, n *PushNotification) error +} +``` + +`PushNotifier` 可先 Noop。媒体：HTTP 凭证 → 对象存储 → 消息只带引用。 + +--- + +# 16. Redis vs MySQL + +| MySQL | Redis | +| --- | --- | +| users / 好友 / 群 / 成员 | 在线、Session | +| messages | deliver:queue / processing | +| conversations / p2p 唯一 | dialog ZSET、未读、预览 | +| last_read_msg_id | 限流 | +| mailbox_events | Snowflake 无关 | + +业务只依赖接口：`IDGenerator`、`InboxStore`、`DeliverQueue`、`PresenceStore`、`RateLimiter`。 + +--- + +# 17. 错误 / 日志 / 指标 + +```go +var ( + ErrUnauthorized, ErrForbidden, ErrNotFound + ErrInvalidArgument, ErrDuplicate + ErrRateLimited, ErrBusy, ErrInternal +) +``` + +日志：`trace_id user_id device_id real_ip connection_id command conversation_id msg_id`。 + +指标： + +```text +tcp_* eventloop_react_seconds +taskpool_depth taskpool_rejected_total +message_send_* persist_latency enqueue_latency send_ack_latency +deliver_queue_depth deliver_processing_depth deliver_job_latency +deliver_idempotent_skip_total +outbound_drop_total push_ack_timeout_total +inbox_sync_total message_sync_total +realip_missing_total # PROXY 头异常 +``` + +--- + +# 18. 启动与关闭 + +```text +Config → Logger → MySQL → Redis → ID + → RequeueStale processing + → Session / TaskPool / Deliver Worker + → gnet 127.0.0.1:9000（PROXY+TLS） + → Gin 127.0.0.1:8080 + → Ready +``` + +SIGTERM：探活失败 → 停 Accept → 抽干 TaskPool → Worker 再 Take 一轮后退出 → 未 Ack 的 processing 留着下次 `RequeueStale` → 关连接 / Redis / MySQL。 + +--- + +# 19. 配置 + +```yaml +server: + http: + addr: "127.0.0.1:8080" + tcp: + addr: "127.0.0.1:9000" + proxy_protocol: true + tls: + cert_file: "/etc/im/im.example.com.crt" + key_file: "/etc/im/im.example.com.key" + max_frame_size: 65536 + idle_timeout: 120s + keepalive: 30s + recv_buffer: 16777216 + send_buffer: 16777216 + multicore: true + compress_min_size: 512 + +id: + snowflake_worker_id: 1 + +mysql: + dsn: "..." + +redis: + addr: "127.0.0.1:6379" + deliver_queue_key: "deliver:queue" + deliver_processing_key: "deliver:processing" + +taskpool: + size: 256 + queue: 1024 + +deliver: + workers: 8 + take_timeout: 5s + done_ttl: 10m + +rate_limit: + message_per_second: 20 + auth_per_second: 5 + sync_inbox_per_minute: 60 + sync_messages_per_minute: 120 + history_per_minute: 200 + +message: + max_content_size: 65536 + sync_limit: 200 + msgid_expire_ms: 300000 +``` + +--- + +# 20. 不变量 + +1. `SEND_ACK` = MySQL 已有消息 **且** 已 `Push` 进 deliver 队列 +2. `(sender_id, conversation_id, client_msg_id)` 唯一 +3. 会话内顺序是 `MsgId` +4. P2P `(user_small, user_big)` 唯一对应一个 `conversation_id` +5. 消息正文只在 MySQL；列表 / 未读 / 预览在 Redis +6. mailbox 只用于推送失败和控制事件 +7. EventLoop 不做 MySQL / 扇出 +8. 业务不碰 `gnet.Conn`；限流用 PROXY 真实 IP +9. gnet 只监听回环；公网只开 443；未知 SNI 断开 +10. 保密靠 TLS，不靠应用层加密 +11. 同一 `MsgId` 的 deliver 任务只生效一次 +12. 发送设备不回推 MESSAGE；其他设备必须投递 +13. Redis 丢列表不丢正文；丢队列不推翻已 ACK 的前提是入队已成功 + +--- + +# 21. 核心链路 + +```text +Client TLS :443 + Nginx SNI im.example.com + PROXY + gnet 读真实 IP → TLS → Frame + TaskPool + 权限 → INSERT messages → RPUSH 可靠队列 → SEND_ACK + Deliver Worker + 幂等 → ZSET 列表 → 在线 MESSAGE + 失败 → mailbox + 可选系统推送 +``` + +离线： + +```text +AUTH → SYNC_INBOX(ZSET 差量 + mailbox) → SYNC_MESSAGES(id > last_msg_id) +``` + +HTTP： + +```text +SNI api.example.com → Nginx 终结 TLS → Gin → 同一 Application +``` + +--- + +# 22. 开发顺序 + +1. 基础设施 + Snowflake worker id +2. Nginx 443 SNI + PROXY；Gin / gnet 本机打通；验收真实 IP +3. Frame、压缩、Session、AUTH、心跳超时对齐 +4. P2P 唯一会话、消息落库、幂等、`GENERATE_MSG_ID` +5. 可靠队列：入队后再 ACK +6. Worker：ZSET 列表、预览、未读幂等、在线 Push、失败 mailbox +7. 两级 SYNC + 分项限流 +8. 已读、撤回、免打扰、置顶 +9. Upload +10. 好友 / 拉黑 / 群成员 / 禁言校验 + 群扇出 +11. HTTP 写路径复用 Application +12. PushNotifier（先 Noop） +13. 压测验收 + +列表、邮箱、Upload 必须和消息主路径一起闭环。 + +--- + +# 23. 验收 + +```text +公网只有 443；:8080 / :9000 外部不可达 +SNI api. / im. 分流正确；未知 SNI 断开 +IM 抓包为 TLS；gnet 日志里是客户端真实 IP，不是 127.0.0.1 +心跳 30s、空闲 120s、Nginx 1h：长连接不被静默掐断 +EventLoop P99 不随 MySQL P99 线性恶化 +COMMIT 后 Redis 失败：客户端收不到 SEND_ACK，重试不双发 +ACK 后杀进程：processing 重启后续上，unread 不 +2 +同 client_msg_id / 同 MsgId 返回同一条 +P2P 两人无论谁先发，conversation_id 相同 +千人群：SEND_ACK P99 稳定，MySQL 无 N 行 inbox 更新 +离线用户上线：列表 ZSET 能看见会话，再按 MsgId 拉正文 +撤回 / KICK 经 mailbox 到达；成功在线 Push 不写 mailbox +发送方第二台设备能收到 +慢客户端 outbound 满不拖垮进程 +清空 Redis：历史消息仍在；列表可重建；不出现 MsgId 回绕 +``` + +--- + +# 24. 何时再加复杂度 + +| 现象 | 才引入 | +| --- | --- | +| 单机 443 / CPU / 连接打满 | 多机 + 上游 4 层 | +| deliver 持续堆积 | 加 worker；再按 conversation 分 list | +| 大群 / 搜索 / 审计削峰 | MQ | +| 模块独立故障域 | 拆服务 | + +--- + +# 25. 一句话 + +> **一台机器、一个 443；Nginx 按 SNI 拆 HTTP 与 IM，并用 PROXY 把头上的真 IP 传给 gnet；gnet 在本机做 TLS 和薄 Frame；EventLoop 不碰 MySQL；消息用 MySQL + Snowflake MsgId；先可靠入队再 ACK；会话列表走 Redis ZSET；邮箱只收失败推送和控制事件；最终靠两级 SYNC 补齐。** diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..7afc08c --- /dev/null +++ b/go.mod @@ -0,0 +1,53 @@ +module spark-server + +go 1.26.0 + +require ( + github.com/bwmarrin/snowflake v0.3.0 + github.com/gin-gonic/gin v1.12.0 + github.com/go-sql-driver/mysql v1.10.1 + github.com/panjf2000/gnet/v2 v2.10.0 + github.com/redis/go-redis/v9 v9.22.0 + google.golang.org/protobuf v1.36.10 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + filippo.io/edwards25519 v1.2.0 // indirect + github.com/bytedance/gopkg v0.1.3 // indirect + github.com/bytedance/sonic v1.15.0 // indirect + github.com/bytedance/sonic/loader v0.5.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cloudwego/base64x v0.1.6 // indirect + github.com/gabriel-vasile/mimetype v1.4.12 // indirect + github.com/gin-contrib/sse v1.1.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.30.1 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/goccy/go-yaml v1.19.2 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/panjf2000/ants/v2 v2.12.1 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/quic-go/qpack v0.6.0 // indirect + github.com/quic-go/quic-go v0.59.0 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.3.1 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.28.0 // indirect + golang.org/x/arch v0.22.0 // indirect + golang.org/x/crypto v0.48.0 // indirect + golang.org/x/net v0.51.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/sys v0.41.0 // indirect + golang.org/x/text v0.34.0 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..85a3d86 --- /dev/null +++ b/go.sum @@ -0,0 +1,133 @@ +filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= +filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0= +github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE= +github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= +github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= +github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= +github.com/bytedance/sonic v1.15.0/go.mod h1:tFkWrPz0/CUCLEF4ri4UkHekCIcdnkqXw9VduqpJh0k= +github.com/bytedance/sonic/loader v0.5.0 h1:gXH3KVnatgY7loH5/TkeVyXPfESoqSBSBEiDd5VjlgE= +github.com/bytedance/sonic/loader v0.5.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= +github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gabriel-vasile/mimetype v1.4.12 h1:e9hWvmLYvtp846tLHam2o++qitpguFiYCKbn0w9jyqw= +github.com/gabriel-vasile/mimetype v1.4.12/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= +github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= +github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= +github.com/gin-gonic/gin v1.12.0 h1:b3YAbrZtnf8N//yjKeU2+MQsh2mY5htkZidOM7O0wG8= +github.com/gin-gonic/gin v1.12.0/go.mod h1:VxccKfsSllpKshkBWgVgRniFFAzFb9csfngsqANjnLc= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w= +github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM= +github.com/go-sql-driver/mysql v1.10.1 h1:arlSnNLq6a5yxGxV7qg9lF4j0C+KwD6NbQyKr9QL6ME= +github.com/go-sql-driver/mysql v1.10.1/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= +github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/panjf2000/ants/v2 v2.12.1 h1:BWvU2wHpyXWxhhNXsGB6JXLCNbshyLd1QxvoAmZnu10= +github.com/panjf2000/ants/v2 v2.12.1/go.mod h1:tSQuaNQ6r6NRhPt+IZVUevvDyFMTs+eS4ztZc52uJTY= +github.com/panjf2000/gnet/v2 v2.10.0 h1:rC4jNF+jtXj/FH+8JOIQ3XxjD+yBunYBLKg9TE3dc4g= +github.com/panjf2000/gnet/v2 v2.10.0/go.mod h1:f9wdbOFsdbZqlSvXctWbPRW5bB/W++q8Zqz+D7tQIVQ= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= +github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= +github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw= +github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= +github.com/redis/go-redis/v9 v9.22.0 h1:laDvpYXTJtZLloinw1fA5Kqd6HAEH2XKxOkG/PDq2F0= +github.com/redis/go-redis/v9 v9.22.0/go.mod h1:y2g0Wj8rQvuK0ELM+oxSudcLtC09JScs98I/X9gRWY4= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY= +github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= +go.mongodb.org/mongo-driver/v2 v2.5.0 h1:yXUhImUjjAInNcpTcAlPHiT7bIXhshCTL3jVBkF3xaE= +go.mongodb.org/mongo-driver/v2 v2.5.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= +go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= +go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/arch v0.22.0 h1:c/Zle32i5ttqRXjdLyyHZESLD/bB90DCU1g9l/0YBDI= +golang.org/x/arch v0.22.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= +golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= +golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= +golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= +golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/bootstrap/app.go b/internal/bootstrap/app.go new file mode 100644 index 0000000..5cd944a --- /dev/null +++ b/internal/bootstrap/app.go @@ -0,0 +1,125 @@ +package bootstrap + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "log/slog" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "spark-server/internal/infrastructure/id" + "spark-server/internal/infrastructure/logger" + "spark-server/internal/infrastructure/mysql" + redisx "spark-server/internal/infrastructure/redis" + httpx "spark-server/internal/transport/http" + tcp "spark-server/internal/transport/tcp" + "spark-server/internal/transport/tcp/session" +) + +type App struct { + cfg *Config + log *slog.Logger + http *httpx.Server + tcp *tcp.Server + pool tcp.TaskPool + mysql *mysql.DB + redis *redisx.Client + idgen *id.Generator + sessions session.Manager +} + +func New(cfg *Config) (*App, error) { + log := logger.New() + idgen, err := id.New(cfg.ID.SnowflakeWorkerID) + if err != nil { + return nil, err + } + db, err := mysql.Open(cfg.MySQL.DSN) + if err != nil { + return nil, err + } + rdb, err := redisx.Open(cfg.Redis.Addr) + if err != nil { + _ = db.Close() + return nil, err + } + pool := tcp.NewTaskPool(cfg.TaskPool.Size, cfg.TaskPool.Queue) + sessions := session.NewManager() + tlsConf, err := loadTLS(cfg.Server.TCP.TLS) + if err != nil { + return nil, err + } + tcpSrv := tcp.NewServer(log, pool, sessions, &tcp.Options{ + Addr: cfg.Server.TCP.Addr, + Multicore: cfg.Server.TCP.Multicore, + KeepAlive: cfg.Server.TCP.KeepAlive, + RecvBuffer: cfg.Server.TCP.RecvBuffer, + SendBuffer: cfg.Server.TCP.SendBuffer, + MaxFrameSize: cfg.Server.TCP.MaxFrameSize, + TLS: tlsConf, + ProxyProtocol: cfg.Server.TCP.ProxyProtocol, + IdleTimeout: cfg.Server.TCP.IdleTimeout, + OutboundSize: cfg.Session.OutboundQueueSize, + RatePerSec: cfg.RateLimit.MessagePerSecond, + CompressMin: cfg.Server.TCP.CompressMinSize, + IDs: idgen, + }) + return &App{ + cfg: cfg, + log: log, + http: httpx.New(log, cfg.Server.HTTP.Addr), + tcp: tcpSrv, + pool: pool, + mysql: db, + redis: rdb, + idgen: idgen, + sessions: sessions, + }, nil +} + +func loadTLS(t TLSConfig) (*tls.Config, error) { + if t.CertFile == "" || t.KeyFile == "" { + return nil, nil + } + cert, err := tls.LoadX509KeyPair(t.CertFile, t.KeyFile) + if err != nil { + return nil, fmt.Errorf("tls: %w", err) + } + return &tls.Config{MinVersion: tls.VersionTLS12, Certificates: []tls.Certificate{cert}}, nil +} + +func (a *App) Run() error { + errCh := make(chan error, 2) + go func() { errCh <- a.http.Start() }() + go func() { errCh <- a.tcp.Start() }() + + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + select { + case err := <-errCh: + return err + case sig := <-sigCh: + a.log.Info("shutdown", "signal", sig.String()) + return a.Shutdown() + } +} + +func (a *App) Shutdown() error { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + var err error + if e := a.http.Stop(ctx); e != nil && !errors.Is(e, http.ErrServerClosed) { + err = errors.Join(err, e) + } + if e := a.tcp.Stop(ctx); e != nil { + err = errors.Join(err, e) + } + a.pool.Close() + err = errors.Join(err, a.redis.Close(), a.mysql.Close()) + return err +} diff --git a/internal/bootstrap/config.go b/internal/bootstrap/config.go new file mode 100644 index 0000000..3f78e8f --- /dev/null +++ b/internal/bootstrap/config.go @@ -0,0 +1,104 @@ +// Package bootstrap loads config and owns process lifecycle. +package bootstrap + +import ( + "fmt" + "os" + "time" + + "gopkg.in/yaml.v3" +) + +type Config struct { + Server ServerConfig `yaml:"server"` + ID IDConfig `yaml:"id"` + MySQL MySQLConfig `yaml:"mysql"` + Redis RedisConfig `yaml:"redis"` + TaskPool TaskPoolConfig `yaml:"taskpool"` + Deliver DeliverConfig `yaml:"deliver"` + Session SessionConfig `yaml:"session"` + RateLimit RateLimitConfig `yaml:"rate_limit"` + Message MessageConfig `yaml:"message"` +} + +type ServerConfig struct { + HTTP HTTPConfig `yaml:"http"` + TCP TCPConfig `yaml:"tcp"` +} + +type HTTPConfig struct { + Addr string `yaml:"addr"` +} + +type TCPConfig struct { + Addr string `yaml:"addr"` + ProxyProtocol bool `yaml:"proxy_protocol"` + TLS TLSConfig `yaml:"tls"` + MaxFrameSize int `yaml:"max_frame_size"` + IdleTimeout time.Duration `yaml:"idle_timeout"` + KeepAlive time.Duration `yaml:"keepalive"` + RecvBuffer int `yaml:"recv_buffer"` + SendBuffer int `yaml:"send_buffer"` + Multicore bool `yaml:"multicore"` + CompressMinSize int `yaml:"compress_min_size"` +} + +type TLSConfig struct { + CertFile string `yaml:"cert_file"` + KeyFile string `yaml:"key_file"` +} + +type IDConfig struct { + SnowflakeWorkerID int64 `yaml:"snowflake_worker_id"` +} + +type MySQLConfig struct { + DSN string `yaml:"dsn"` +} + +type RedisConfig struct { + Addr string `yaml:"addr"` + DeliverQueueKey string `yaml:"deliver_queue_key"` + DeliverProcessingKey string `yaml:"deliver_processing_key"` +} + +type TaskPoolConfig struct { + Size int `yaml:"size"` + Queue int `yaml:"queue"` +} + +type DeliverConfig struct { + Workers int `yaml:"workers"` + TakeTimeout time.Duration `yaml:"take_timeout"` + DoneTTL time.Duration `yaml:"done_ttl"` +} + +type SessionConfig struct { + OutboundQueueSize int `yaml:"outbound_queue_size"` +} + +type RateLimitConfig struct { + MessagePerSecond int `yaml:"message_per_second"` + AuthPerSecond int `yaml:"auth_per_second"` + SyncInboxPerMinute int `yaml:"sync_inbox_per_minute"` + SyncMessagesPerMinute int `yaml:"sync_messages_per_minute"` + HistoryPerMinute int `yaml:"history_per_minute"` +} + +type MessageConfig struct { + MaxContentSize int `yaml:"max_content_size"` + SyncLimit int `yaml:"sync_limit"` + MsgIDExpireMS int64 `yaml:"msgid_expire_ms"` +} + +func Load(path string) (*Config, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read config: %w", err) + } + cfg := &Config{} + if err := yaml.Unmarshal(raw, cfg); err != nil { + return nil, fmt.Errorf("parse config: %w", err) + } + return cfg, nil +} diff --git a/internal/infrastructure/id/id.go b/internal/infrastructure/id/id.go new file mode 100644 index 0000000..b229669 --- /dev/null +++ b/internal/infrastructure/id/id.go @@ -0,0 +1,41 @@ +package id + +import ( + "fmt" + "sync" + "time" + + "github.com/bwmarrin/snowflake" + + "spark-server/internal/pkg/errs" +) + +// Generator issues Snowflake MsgIds. Worker ID is fixed per machine. Clock rewind is rejected. +type Generator struct { + mu sync.Mutex + node *snowflake.Node + lastMs int64 + workerID int64 +} + +func New(workerID int64) (*Generator, error) { + node, err := snowflake.NewNode(workerID) + if err != nil { + return nil, fmt.Errorf("snowflake node: %w", err) + } + return &Generator{node: node, workerID: workerID}, nil +} + +func (g *Generator) NextMsgID() (int64, error) { + g.mu.Lock() + defer g.mu.Unlock() + now := time.Now().UnixMilli() + if now < g.lastMs { + return 0, errs.ErrClockRewind + } + id := g.node.Generate().Int64() + g.lastMs = now + return id, nil +} + +func (g *Generator) WorkerID() int64 { return g.workerID } diff --git a/internal/infrastructure/logger/logger.go b/internal/infrastructure/logger/logger.go new file mode 100644 index 0000000..4428a03 --- /dev/null +++ b/internal/infrastructure/logger/logger.go @@ -0,0 +1,10 @@ +package logger + +import ( + "log/slog" + "os" +) + +func New() *slog.Logger { + return slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo})) +} diff --git a/internal/infrastructure/metrics/metrics.go b/internal/infrastructure/metrics/metrics.go new file mode 100644 index 0000000..a04ce5f --- /dev/null +++ b/internal/infrastructure/metrics/metrics.go @@ -0,0 +1,9 @@ +package metrics + +// Nop is a placeholder until Prometheus hooks are wired. +type Nop struct{} + +func NewNop() *Nop { return &Nop{} } + +func (*Nop) Inc(_ string) {} +func (*Nop) Observe(_ string, _ float64) {} diff --git a/internal/infrastructure/mysql/mysql.go b/internal/infrastructure/mysql/mysql.go new file mode 100644 index 0000000..01b0794 --- /dev/null +++ b/internal/infrastructure/mysql/mysql.go @@ -0,0 +1,45 @@ +package mysql + +import ( + "context" + "database/sql" + "fmt" + "time" + + // registers the MySQL driver with database/sql + _ "github.com/go-sql-driver/mysql" +) + +type DB struct { + sql *sql.DB +} + +func Open(dsn string) (*DB, error) { + if dsn == "" { + return &DB{}, nil + } + sqlDB, err := sql.Open("mysql", dsn) + if err != nil { + return nil, fmt.Errorf("mysql open: %w", err) + } + sqlDB.SetMaxOpenConns(64) + sqlDB.SetConnMaxLifetime(30 * time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + if err := sqlDB.PingContext(ctx); err != nil { + _ = sqlDB.Close() + return nil, fmt.Errorf("mysql ping: %w", err) + } + return &DB{sql: sqlDB}, nil +} + +func (d *DB) SQL() *sql.DB { return d.sql } + +func (d *DB) Close() error { + if d == nil || d.sql == nil { + return nil + } + return d.sql.Close() +} + +func (d *DB) Ready() bool { return d != nil && d.sql != nil } diff --git a/internal/infrastructure/push/push.go b/internal/infrastructure/push/push.go new file mode 100644 index 0000000..86c2ae1 --- /dev/null +++ b/internal/infrastructure/push/push.go @@ -0,0 +1,23 @@ +package push + +import "context" + +// Notification is a system push. Mute is decided in Deliver Worker, not gnet. +type Notification struct { + UserID int64 + DeviceID uint64 + ConversationID int64 + Title string + Body string + Badge int + CollapseKey string +} + +type Notifier interface { + Notify(ctx context.Context, n *Notification) error +} + +// Noop satisfies V3: the interface exists; APNS/FCM can wait. +type Noop struct{} + +func (Noop) Notify(context.Context, *Notification) error { return nil } diff --git a/internal/infrastructure/redis/redis.go b/internal/infrastructure/redis/redis.go new file mode 100644 index 0000000..e69d067 --- /dev/null +++ b/internal/infrastructure/redis/redis.go @@ -0,0 +1,38 @@ +package redisx + +import ( + "context" + "fmt" + "time" + + "github.com/redis/go-redis/v9" +) + +type Client struct { + rdb *redis.Client +} + +func Open(addr string) (*Client, error) { + if addr == "" { + return &Client{}, nil + } + rdb := redis.NewClient(&redis.Options{Addr: addr}) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + if err := rdb.Ping(ctx).Err(); err != nil { + _ = rdb.Close() + return nil, fmt.Errorf("redis ping: %w", err) + } + return &Client{rdb: rdb}, nil +} + +func (c *Client) RDB() *redis.Client { return c.rdb } + +func (c *Client) Close() error { + if c == nil || c.rdb == nil { + return nil + } + return c.rdb.Close() +} + +func (c *Client) Ready() bool { return c != nil && c.rdb != nil } diff --git a/internal/module/auth/service.go b/internal/module/auth/service.go new file mode 100644 index 0000000..a48d6c9 --- /dev/null +++ b/internal/module/auth/service.go @@ -0,0 +1,12 @@ +package auth + +import "context" + +type Principal struct { + UserID int64 + DeviceID uint64 +} + +type Service interface { + Authenticate(ctx context.Context, token string) (*Principal, error) +} diff --git a/internal/module/auth/static.go b/internal/module/auth/static.go new file mode 100644 index 0000000..c17cb93 --- /dev/null +++ b/internal/module/auth/static.go @@ -0,0 +1,21 @@ +package auth + +import ( + "context" + + "spark-server/internal/pkg/errs" +) + +// Static accepts one token. Used in tests and local bring-up. +type Static struct { + Token string + UserID int64 + DeviceID uint64 +} + +func (s Static) Authenticate(_ context.Context, token string) (*Principal, error) { + if s.Token == "" || token != s.Token { + return nil, errs.ErrUnauthorized + } + return &Principal{UserID: s.UserID, DeviceID: s.DeviceID}, nil +} diff --git a/internal/module/conversation/repository.go b/internal/module/conversation/repository.go new file mode 100644 index 0000000..1fdff57 --- /dev/null +++ b/internal/module/conversation/repository.go @@ -0,0 +1,22 @@ +package conversation + +import "context" + +const ( + TypeP2P int = 1 + TypeGroup int = 2 +) + +type Conversation struct { + ID int64 + Type int + LastMsgID int64 + LastMsgAt int64 +} + +type Repository interface { + Create(ctx context.Context, c *Conversation) error + Get(ctx context.Context, id int64) (*Conversation, error) + GetP2P(ctx context.Context, userA, userB int64) (*Conversation, error) + UpdateLastMsg(ctx context.Context, id, msgID, atMS int64) error +} diff --git a/internal/module/deliver/queue.go b/internal/module/deliver/queue.go new file mode 100644 index 0000000..907404e --- /dev/null +++ b/internal/module/deliver/queue.go @@ -0,0 +1,26 @@ +package deliver + +import ( + "context" + "time" +) + +// Job is a fanout unit. ExcludeDevice is the sender device. NotMailbox matches BFF NotSaveIch. +type Job struct { + MsgID int64 + Cmd uint16 + ConversationID int64 + FromUserID int64 + FromDeviceID uint64 + ExcludeDevice uint64 + NotPush bool + NotMailbox bool + Payload []byte +} + +type Queue interface { + Push(ctx context.Context, job *Job) error + Take(ctx context.Context, timeout time.Duration) (*Job, error) + Ack(ctx context.Context, job *Job) error + RequeueStale(ctx context.Context) error +} diff --git a/internal/module/deliver/redis_queue.go b/internal/module/deliver/redis_queue.go new file mode 100644 index 0000000..3ec3a2b --- /dev/null +++ b/internal/module/deliver/redis_queue.go @@ -0,0 +1,78 @@ +package deliver + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/redis/go-redis/v9" + + "spark-server/internal/pkg/errs" +) + +type redisQueue struct { + rdb *redis.Client + queueKey string + processKey string +} + +func NewRedisQueue(rdb *redis.Client, queueKey, processKey string) Queue { + return &redisQueue{rdb: rdb, queueKey: queueKey, processKey: processKey} +} + +func (q *redisQueue) Push(ctx context.Context, job *Job) error { + if q.rdb == nil { + return errs.ErrInternal + } + raw, err := json.Marshal(job) + if err != nil { + return err + } + return q.rdb.RPush(ctx, q.queueKey, raw).Err() +} + +func (q *redisQueue) Take(ctx context.Context, timeout time.Duration) (*Job, error) { + if q.rdb == nil { + return nil, errs.ErrInternal + } + res, err := q.rdb.BRPopLPush(ctx, q.queueKey, q.processKey, timeout).Result() + if err == redis.Nil { + return nil, nil + } + if err != nil { + return nil, err + } + job := &Job{} + if err := json.Unmarshal([]byte(res), job); err != nil { + return nil, fmt.Errorf("deliver job: %w", err) + } + return job, nil +} + +func (q *redisQueue) Ack(ctx context.Context, job *Job) error { + if q.rdb == nil || job == nil { + return nil + } + raw, err := json.Marshal(job) + if err != nil { + return err + } + return q.rdb.LRem(ctx, q.processKey, 1, raw).Err() +} + +func (q *redisQueue) RequeueStale(ctx context.Context) error { + if q.rdb == nil { + return nil + } + for { + res, err := q.rdb.RPopLPush(ctx, q.processKey, q.queueKey).Result() + if err == redis.Nil { + return nil + } + if err != nil { + return err + } + _ = res + } +} diff --git a/internal/module/friend/service.go b/internal/module/friend/service.go new file mode 100644 index 0000000..60b5b4d --- /dev/null +++ b/internal/module/friend/service.go @@ -0,0 +1,7 @@ +package friend + +import "context" + +type Service interface { + CanSend(ctx context.Context, fromUserID, toUserID int64) error +} diff --git a/internal/module/group/service.go b/internal/module/group/service.go new file mode 100644 index 0000000..cbe4309 --- /dev/null +++ b/internal/module/group/service.go @@ -0,0 +1,8 @@ +package group + +import "context" + +type Service interface { + CanSend(ctx context.Context, userID, groupID int64) error + MemberIDs(ctx context.Context, groupID int64) ([]int64, error) +} diff --git a/internal/module/inbox/store.go b/internal/module/inbox/store.go new file mode 100644 index 0000000..ebcaafe --- /dev/null +++ b/internal/module/inbox/store.go @@ -0,0 +1,12 @@ +package inbox + +import "context" + +// Store is the Redis ZSET conversation list. MySQL is not on this hot path. +type Store interface { + TouchUnread(ctx context.Context, userID, conversationID int64, score int64) error + TouchRead(ctx context.Context, userID, conversationID int64, score int64) error + IncrUnread(ctx context.Context, userID, conversationID int64, delta int64) error + SetPreview(ctx context.Context, conversationID int64, preview []byte) error + ListUpdatedSince(ctx context.Context, userID, score int64, limit int) ([]int64, error) +} diff --git a/internal/module/message/service.go b/internal/module/message/service.go new file mode 100644 index 0000000..1595cfe --- /dev/null +++ b/internal/module/message/service.go @@ -0,0 +1,43 @@ +package message + +import "context" + +type Message struct { + ID int64 + ConversationID int64 + SenderID int64 + ClientMsgID string + Type int + Content []byte + Status int + CreatedAtMS int64 +} + +type Repository interface { + Create(ctx context.Context, message *Message) error + GetByID(ctx context.Context, id int64) (*Message, error) + GetByClientMsgID(ctx context.Context, senderID, conversationID int64, clientMsgID string) (*Message, error) + ListByConversation(ctx context.Context, conversationID, lastMsgID int64, limit int) ([]*Message, error) +} + +type SendRequest struct { + ConversationID int64 + SenderID int64 + DeviceID uint64 + ClientMsgID string + MsgID int64 + Type int + Content []byte +} + +type SendResponse struct { + MsgID int64 + ClientMsgID string + ServerTimeMS int64 +} + +// Service owns persist + enqueue. SEND_ACK happens only after both succeed. +type Service interface { + GenerateMsgID(ctx context.Context) (int64, error) + SendMessage(ctx context.Context, req *SendRequest) (*SendResponse, error) +} diff --git a/internal/module/session/policy.go b/internal/module/session/policy.go new file mode 100644 index 0000000..bcb573b --- /dev/null +++ b/internal/module/session/policy.go @@ -0,0 +1,5 @@ +package sessionmod + +// Policy is owned by the session module, not gnet. +// Same device_id kicks the old connection. Other devices of the user stay. +type Policy struct{} diff --git a/internal/module/sync/service.go b/internal/module/sync/service.go new file mode 100644 index 0000000..fda3b00 --- /dev/null +++ b/internal/module/sync/service.go @@ -0,0 +1,21 @@ +package syncx + +import "context" + +type InboxRequest struct { + UserID int64 + DeviceID uint64 + MailboxSeq int64 + DialogScore int64 +} + +type MessagesRequest struct { + ConversationID int64 + LastMsgID int64 + Limit int +} + +type Service interface { + SyncInbox(ctx context.Context, req *InboxRequest) error + SyncMessages(ctx context.Context, req *MessagesRequest) error +} diff --git a/internal/module/upload/service.go b/internal/module/upload/service.go new file mode 100644 index 0000000..9e10e21 --- /dev/null +++ b/internal/module/upload/service.go @@ -0,0 +1,12 @@ +package upload + +import "context" + +type Credential struct { + ObjectKey string + UploadURL string +} + +type Service interface { + CreateCredential(ctx context.Context, userID int64, mime string, size int64) (*Credential, error) +} diff --git a/internal/module/user/service.go b/internal/module/user/service.go new file mode 100644 index 0000000..ae5c53d --- /dev/null +++ b/internal/module/user/service.go @@ -0,0 +1,12 @@ +package user + +import "context" + +type User struct { + ID int64 + Nickname string +} + +type Service interface { + Get(ctx context.Context, id int64) (*User, error) +} diff --git a/internal/pkg/errs/errs.go b/internal/pkg/errs/errs.go new file mode 100644 index 0000000..803ab70 --- /dev/null +++ b/internal/pkg/errs/errs.go @@ -0,0 +1,22 @@ +// Package errs is the shared domain error set. Transport maps these to protocol codes. +package errs + +import "errors" + +var ( + ErrUnauthorized = errors.New("unauthorized") + ErrNotAuthenticated = errors.New("not authenticated") + ErrForbidden = errors.New("forbidden") + ErrNotFound = errors.New("not found") + ErrInvalidArgument = errors.New("invalid argument") + ErrDuplicate = errors.New("duplicate") + ErrRateLimited = errors.New("rate limited") + ErrBusy = errors.New("busy") + ErrInternal = errors.New("internal") + ErrFrameTooLarge = errors.New("frame too large") + ErrInvalidFrame = errors.New("invalid frame") + ErrProxyHeader = errors.New("invalid proxy protocol header") + ErrClockRewind = errors.New("snowflake clock rewind") + ErrClosed = errors.New("connection closed") + ErrOutboundDropped = errors.New("outbound dropped") +) diff --git a/internal/pkg/reqctx/reqctx.go b/internal/pkg/reqctx/reqctx.go new file mode 100644 index 0000000..b35a924 --- /dev/null +++ b/internal/pkg/reqctx/reqctx.go @@ -0,0 +1,26 @@ +package reqctx + +import "context" + +type ctxKey struct{} + +// RequestContext is the per-request identity. Use RealIP for rate limit and audit, never 127.0.0.1 from gnet.RemoteAddr. +type RequestContext struct { + TraceID string + UserID int64 + DeviceID uint64 + Language string + AppVersion string + ConnectionID string + RemoteAddr string + RealIP string +} + +func With(ctx context.Context, rc *RequestContext) context.Context { + return context.WithValue(ctx, ctxKey{}, rc) +} + +func From(ctx context.Context) *RequestContext { + rc, _ := ctx.Value(ctxKey{}).(*RequestContext) + return rc +} diff --git a/internal/transport/http/server.go b/internal/transport/http/server.go new file mode 100644 index 0000000..ef8fbc6 --- /dev/null +++ b/internal/transport/http/server.go @@ -0,0 +1,45 @@ +package httpx + +import ( + "context" + "log/slog" + "net/http" + "time" + + "github.com/gin-gonic/gin" +) + +type Server struct { + log *slog.Logger + http *http.Server +} + +func New(log *slog.Logger, addr string) *Server { + gin.SetMode(gin.ReleaseMode) + r := gin.New() + r.Use(gin.Recovery()) + r.GET("/healthz", func(c *gin.Context) { + c.JSON(http.StatusOK, gin.H{"status": "ok"}) + }) + return &Server{ + log: log, + http: &http.Server{ + Addr: addr, + Handler: r, + ReadHeaderTimeout: 5 * time.Second, + }, + } +} + +func (s *Server) Start() error { + s.log.Info("http listening", "addr", s.http.Addr) + err := s.http.ListenAndServe() + if err == http.ErrServerClosed { + return nil + } + return err +} + +func (s *Server) Stop(ctx context.Context) error { + return s.http.Shutdown(ctx) +} diff --git a/internal/transport/tcp/command_router.go b/internal/transport/tcp/command_router.go new file mode 100644 index 0000000..4f3602c --- /dev/null +++ b/internal/transport/tcp/command_router.go @@ -0,0 +1,36 @@ +package tcp + +import ( + "context" + + "spark-server/internal/pkg/errs" + "spark-server/internal/transport/tcp/protocol" + "spark-server/internal/transport/tcp/session" +) + +// CommandHandler runs in TaskPool for AUTH/SEND/SYNC and other business commands. +type CommandHandler func(ctx context.Context, conn *session.Conn, frame protocol.Frame) (*protocol.Frame, error) + +// Router maps Frame.Command to a handler. +type Router struct { + handlers map[uint16]CommandHandler +} + +// NewRouter creates an empty command table. +func NewRouter() *Router { + return &Router{handlers: make(map[uint16]CommandHandler)} +} + +// Register binds a command to a TaskPool handler. +func (r *Router) Register(cmd uint16, h CommandHandler) { + r.handlers[cmd] = h +} + +// Dispatch looks up and runs the handler. Unknown commands are invalid. +func (r *Router) Dispatch(ctx context.Context, conn *session.Conn, frame protocol.Frame) (*protocol.Frame, error) { + h, ok := r.handlers[frame.Command] + if !ok { + return nil, errs.ErrInvalidArgument + } + return h(ctx, conn, frame) +} diff --git a/internal/transport/tcp/connstate.go b/internal/transport/tcp/connstate.go new file mode 100644 index 0000000..2e4909d --- /dev/null +++ b/internal/transport/tcp/connstate.go @@ -0,0 +1,18 @@ +package tcp + +import ( + "github.com/panjf2000/gnet/v2" + + "spark-server/internal/transport/tcp/session" +) + +type connState struct { + sess *session.Conn + gc gnet.Conn + limit *limiter +} + +func connFrom(c gnet.Conn) *connState { + st, _ := c.Context().(*connState) + return st +} diff --git a/internal/transport/tcp/consume.go b/internal/transport/tcp/consume.go new file mode 100644 index 0000000..3142e46 --- /dev/null +++ b/internal/transport/tcp/consume.go @@ -0,0 +1,97 @@ +package tcp + +import ( + "errors" + "io" + "net" + "strconv" + + "github.com/panjf2000/gnet/v2" + + "spark-server/internal/transport/tcp/protocol" +) + +func (s *Server) consumeProxy(st *connState) gnet.Action { + if !s.proxyProtocol || st.sess.ProxyDone() { + return gnet.None + } + avail := st.gc.InboundBuffered() + if avail == 0 { + return gnet.None + } + buf, err := st.gc.Peek(avail) + if err != nil { + return gnet.Close + } + h, n, err := protocol.ParsePrefix(buf) + if protocol.NeedMore(err) { + return gnet.None + } + if err != nil { + return proxyFallback(st, buf) + } + _, _ = st.gc.Discard(n) + st.sess.SetRealIP(headerRealIP(h, st.sess.RemoteAddr())) + return gnet.None +} + +func proxyFallback(st *connState, buf []byte) gnet.Action { + if protocol.LooksLikeFrame(buf) { + st.sess.MarkProxySkipped() + return gnet.None + } + if protocol.MaybeFramePrefix(buf) && len(buf) < 2 { + return gnet.None + } + return gnet.Close +} + +func headerRealIP(h protocol.Header, fallback string) string { + if h.SrcIP == nil { + return fallback + } + return net.JoinHostPort(h.SrcIP.String(), strconv.Itoa(h.SrcPort)) +} + +func (s *Server) consumeFrame(st *connState) (gnet.Action, bool) { + if st.gc.InboundBuffered() < protocol.HeaderSize { + return gnet.None, false + } + header, err := st.gc.Peek(protocol.HeaderSize) + if err != nil { + if errors.Is(err, io.ErrShortBuffer) { + return gnet.None, false + } + return gnet.Close, false + } + bodyLen, err := protocol.PeekBodyLength(header) + if err != nil || bodyLen > s.maxFrame { + return gnet.Close, false + } + total := protocol.HeaderSize + bodyLen + if st.gc.InboundBuffered() < total { + return gnet.None, false + } + raw, err := st.gc.Peek(total) + if err != nil { + if errors.Is(err, io.ErrShortBuffer) { + return gnet.None, false + } + return gnet.Close, false + } + frame, err := protocol.Decode(raw, s.maxFrame) + if err != nil { + return gnet.Close, false + } + _, _ = st.gc.Discard(total) + body, err := protocol.DecodeBody(frame.Body, frame.Flags, s.maxFrame) + if err != nil { + return gnet.Close, false + } + frame.Body = body + if frame.Version != 0 && frame.Version != protocol.CurrentVersion { + return gnet.Close, false + } + s.handleInbound(st, frame) + return gnet.None, true +} diff --git a/internal/transport/tcp/dispatch.go b/internal/transport/tcp/dispatch.go new file mode 100644 index 0000000..b955af5 --- /dev/null +++ b/internal/transport/tcp/dispatch.go @@ -0,0 +1,93 @@ +package tcp + +import ( + "context" + + "spark-server/internal/pkg/errs" + "spark-server/internal/pkg/reqctx" + "spark-server/internal/transport/tcp/protocol" + "spark-server/internal/transport/tcp/session" +) + +func (s *Server) handleInbound(st *connState, frame protocol.Frame) { + st.sess.Touch() + if frame.Command != protocol.CmdHeartbeat && frame.Command != protocol.CmdPushAck && !st.limit.Allow() { + s.metrics.Inc("tcp_rate_limited_total") + s.replyErr(st, frame, errs.ErrRateLimited) + return + } + if err := ensureAuth(st.sess, frame.Command); err != nil { + s.replyErr(st, frame, err) + return + } + switch frame.Command { + case protocol.CmdHeartbeat: + s.writeEncoded(st, replyFrame(protocol.CmdHeartbeat, frame.SequenceID, frame.RequestID, nil, 0)) + case protocol.CmdPushAck: + return + default: + s.submitBusiness(st, frame) + } +} + +func ensureAuth(conn *session.Conn, cmd uint16) error { + switch cmd { + case protocol.CmdAuth, protocol.CmdHeartbeat, protocol.CmdPushAck: + return nil + default: + if !conn.Authenticated() { + return errs.ErrNotAuthenticated + } + return nil + } +} + +func (s *Server) submitBusiness(st *connState, frame protocol.Frame) { + ctx := reqctx.With(context.Background(), st.sess.RequestContext()) + copied := frame + err := s.pool.Submit(ctx, func() { + s.runHandler(ctx, st.sess, copied) + }) + if err == errs.ErrBusy { + s.metrics.Inc("taskpool_rejected_total") + s.replyErr(st, frame, errs.ErrBusy) + } +} + +func (s *Server) runHandler(ctx context.Context, conn *session.Conn, frame protocol.Frame) { + ctx, cancel := context.WithTimeout(ctx, s.handlerTimeout) + defer cancel() + resp, err := s.router.Dispatch(ctx, conn, frame) + if err != nil { + _ = conn.Send(&session.Packet{ + Command: frame.Command, SequenceID: frame.SequenceID, RequestID: frame.RequestID, + Body: protocol.EncodeError(errorCode(err)), + }) + return + } + if resp == nil { + return + } + _ = conn.Send(&session.Packet{ + Command: resp.Command, SequenceID: resp.SequenceID, RequestID: resp.RequestID, Flags: resp.Flags, Body: resp.Body, + }) +} + +func (s *Server) replyErr(st *connState, frame protocol.Frame, err error) { + body := protocol.EncodeError(errorCode(err)) + s.writeEncoded(st, replyFrame(frame.Command, frame.SequenceID, frame.RequestID, body, 0)) +} + +func (s *Server) writeEncoded(st *connState, frame protocol.Frame) { + body, flags, err := protocol.PrepareBody(frame.Body, s.compressMin) + if err != nil { + return + } + frame.Body = body + frame.Flags |= flags + raw, err := protocol.Encode(frame) + if err != nil { + return + } + _ = st.gc.AsyncWrite(raw, nil) +} diff --git a/internal/transport/tcp/errors.go b/internal/transport/tcp/errors.go new file mode 100644 index 0000000..4567237 --- /dev/null +++ b/internal/transport/tcp/errors.go @@ -0,0 +1,35 @@ +package tcp + +import ( + "errors" + + "spark-server/internal/pkg/errs" + "spark-server/internal/transport/tcp/protocol" +) + +func errorCode(err error) protocol.ErrorBody { + switch { + case err == nil: + return protocol.ErrorBody{Code: protocol.CodeOK} + case errors.Is(err, errs.ErrUnauthorized): + return protocol.ErrorBody{Code: protocol.CodeUnauthorized, Message: err.Error()} + case errors.Is(err, errs.ErrNotAuthenticated): + return protocol.ErrorBody{Code: protocol.CodeNotAuth, Message: err.Error()} + case errors.Is(err, errs.ErrForbidden): + return protocol.ErrorBody{Code: protocol.CodeForbidden, Message: err.Error()} + case errors.Is(err, errs.ErrNotFound): + return protocol.ErrorBody{Code: protocol.CodeNotFound, Message: err.Error()} + case errors.Is(err, errs.ErrInvalidArgument), errors.Is(err, errs.ErrInvalidFrame), errors.Is(err, errs.ErrFrameTooLarge): + return protocol.ErrorBody{Code: protocol.CodeInvalid, Message: err.Error()} + case errors.Is(err, errs.ErrBusy), errors.Is(err, errs.ErrOutboundDropped): + return protocol.ErrorBody{Code: protocol.CodeBusy, Message: err.Error()} + case errors.Is(err, errs.ErrRateLimited): + return protocol.ErrorBody{Code: protocol.CodeRateLimited, Message: err.Error()} + default: + return protocol.ErrorBody{Code: protocol.CodeInternal, Message: "internal"} + } +} + +func replyFrame(cmd uint16, seq uint32, req uint64, body []byte, flags uint16) protocol.Frame { + return protocol.Frame{Magic: protocol.Magic, Version: protocol.CurrentVersion, Command: cmd, Flags: flags, RequestID: req, SequenceID: seq, Body: body} +} diff --git a/internal/transport/tcp/flush.go b/internal/transport/tcp/flush.go new file mode 100644 index 0000000..0aaa6fd --- /dev/null +++ b/internal/transport/tcp/flush.go @@ -0,0 +1,25 @@ +package tcp + +import ( + "spark-server/internal/transport/tcp/protocol" + "spark-server/internal/transport/tcp/session" +) + +func (s *Server) flush(st *connState) { + if st == nil || st.sess == nil { + return + } + for _, p := range st.sess.DrainOutbound(32) { + s.writeEncoded(st, packetFrame(p)) + } +} + +func packetFrame(p *session.Packet) protocol.Frame { + if p == nil { + return protocol.Frame{} + } + return protocol.Frame{ + Magic: protocol.Magic, Version: protocol.CurrentVersion, + Command: p.Command, Flags: p.Flags, RequestID: p.RequestID, SequenceID: p.SequenceID, Body: p.Body, + } +} diff --git a/internal/transport/tcp/handlers.go b/internal/transport/tcp/handlers.go new file mode 100644 index 0000000..251ef68 --- /dev/null +++ b/internal/transport/tcp/handlers.go @@ -0,0 +1,59 @@ +package tcp + +import ( + "context" + + "spark-server/internal/pkg/errs" + "spark-server/internal/transport/tcp/protocol" + "spark-server/internal/transport/tcp/session" +) + +func (s *Server) registerDefaultHandlers() { + s.router.Register(protocol.CmdAuth, s.handleAuth) + s.router.Register(protocol.CmdGenerateMsgID, s.handleGenerateMsgID) +} + +func (s *Server) handleAuth(ctx context.Context, conn *session.Conn, frame protocol.Frame) (*protocol.Frame, error) { + if s.auth == nil { + return nil, errs.ErrUnauthorized + } + req, err := protocol.DecodeAuthRequest(frame.Body) + if err != nil { + return nil, errs.ErrInvalidArgument + } + principal, err := s.auth.Authenticate(ctx, req.Token) + if err != nil { + return nil, err + } + deviceID := principal.DeviceID + if req.DeviceID != 0 { + deviceID = req.DeviceID + } + conn.Authenticate(principal.UserID, deviceID) + if err := s.sessions.Bind(conn); err != nil { + return nil, err + } + conn.Activate() + body := protocol.EncodeAuthResponse(protocol.AuthResponse{UserID: principal.UserID, DeviceID: deviceID}) + return &protocol.Frame{ + Version: protocol.CurrentVersion, Command: protocol.CmdAuth, + RequestID: frame.RequestID, SequenceID: frame.SequenceID, Body: body, + }, nil +} + +func (s *Server) handleGenerateMsgID(_ context.Context, conn *session.Conn, frame protocol.Frame) (*protocol.Frame, error) { + if s.ids == nil { + return nil, errs.ErrInternal + } + if !conn.Authenticated() { + return nil, errs.ErrUnauthorized + } + id, err := s.ids.NextMsgID() + if err != nil { + return nil, err + } + return &protocol.Frame{ + Version: protocol.CurrentVersion, Command: protocol.CmdGenerateMsgID, + RequestID: frame.RequestID, SequenceID: frame.SequenceID, Body: protocol.EncodeMsgID(id), + }, nil +} diff --git a/internal/transport/tcp/limiter.go b/internal/transport/tcp/limiter.go new file mode 100644 index 0000000..cc796c1 --- /dev/null +++ b/internal/transport/tcp/limiter.go @@ -0,0 +1,35 @@ +package tcp + +import "time" + +type limiter struct { + burst float64 + tokens float64 + last time.Time +} + +func newLimiter(perSec int) *limiter { + if perSec < 1 { + perSec = 20 + } + now := time.Now() + return &limiter{burst: float64(perSec), tokens: float64(perSec), last: now} +} + +func (l *limiter) Allow() bool { + if l == nil { + return true + } + now := time.Now() + elapsed := now.Sub(l.last).Seconds() + l.tokens += elapsed * l.burst + if l.tokens > l.burst { + l.tokens = l.burst + } + l.last = now + if l.tokens < 1 { + return false + } + l.tokens-- + return true +} diff --git a/internal/transport/tcp/protocol/codes.go b/internal/transport/tcp/protocol/codes.go new file mode 100644 index 0000000..50a46f6 --- /dev/null +++ b/internal/transport/tcp/protocol/codes.go @@ -0,0 +1,16 @@ +package protocol + +const ( + CodeOK int32 = 0 + CodeUnauthorized int32 = 40101 + CodeForbidden int32 = 40301 + CodeNotFound int32 = 40401 + CodeInvalid int32 = 40001 + CodeBusy int32 = 42901 + CodeRateLimited int32 = 42902 + CodeInternal int32 = 50001 + CodeNotAuth int32 = 40102 + CodeOutboundDrop int32 = 42903 +) + +const CurrentVersion uint8 = 1 diff --git a/internal/transport/tcp/protocol/compress.go b/internal/transport/tcp/protocol/compress.go new file mode 100644 index 0000000..21cadd0 --- /dev/null +++ b/internal/transport/tcp/protocol/compress.go @@ -0,0 +1,61 @@ +package protocol + +import ( + "bytes" + "compress/gzip" + "io" + + "spark-server/internal/pkg/errs" +) + +func Compress(body []byte) ([]byte, error) { + var buf bytes.Buffer + w := gzip.NewWriter(&buf) + if _, err := w.Write(body); err != nil { + _ = w.Close() + return nil, err + } + if err := w.Close(); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func Decompress(body []byte, limit int) ([]byte, error) { + r, err := gzip.NewReader(bytes.NewReader(body)) + if err != nil { + return nil, errs.ErrInvalidFrame + } + defer r.Close() + out, err := io.ReadAll(io.LimitReader(r, int64(limit)+1)) + if err != nil { + return nil, errs.ErrInvalidFrame + } + if len(out) > limit { + return nil, errs.ErrFrameTooLarge + } + return out, nil +} + +// PrepareBody gzip-compresses body when it is at least minCompress bytes and smaller after gzip. +func PrepareBody(body []byte, minCompress int) ([]byte, uint16, error) { + if minCompress <= 0 || len(body) < minCompress { + return body, 0, nil + } + out, err := Compress(body) + if err != nil { + return nil, 0, err + } + if len(out) >= len(body) { + return body, 0, nil + } + return out, FlagCompress, nil +} + +// DecodeBody inflates a gzip body when FlagCompress is set. +func DecodeBody(body []byte, flags uint16, limit int) ([]byte, error) { + if flags&FlagCompress == 0 { + return body, nil + } + return Decompress(body, limit) +} diff --git a/internal/transport/tcp/protocol/frame.go b/internal/transport/tcp/protocol/frame.go new file mode 100644 index 0000000..1a311e4 --- /dev/null +++ b/internal/transport/tcp/protocol/frame.go @@ -0,0 +1,109 @@ +// Package protocol implements TLS-payload framing and PROXY v1/v2 parsing. +package protocol + +import ( + "encoding/binary" + + "spark-server/internal/pkg/errs" +) + +const ( + Magic uint16 = 0x5350 // "SP" + HeaderSize = 23 + FlagCompress = 1 << 0 + + CmdAuth uint16 = 1 + CmdHeartbeat uint16 = 2 + CmdGenerateMsgID uint16 = 3 + CmdSendMessage uint16 = 4 + CmdSendAck uint16 = 5 + CmdMessage uint16 = 6 + CmdPushAck uint16 = 7 + CmdReadMessage uint16 = 8 + CmdRecallMessage uint16 = 9 + CmdSyncInbox uint16 = 10 + CmdSyncInboxFinish uint16 = 11 + CmdSyncMessages uint16 = 12 + CmdSyncMessagesFinish uint16 = 13 + CmdKick uint16 = 14 +) + +// Frame is the application header after TLS. SequenceID matches request/response and PUSH_ACK only. +type Frame struct { + Magic uint16 + Version uint8 + Command uint16 + Flags uint16 + RequestID uint64 + SequenceID uint32 + Body []byte +} + +func Encode(f Frame) ([]byte, error) { + if len(f.Body) > 0xFFFFFFFF { + return nil, errs.ErrFrameTooLarge + } + buf := make([]byte, HeaderSize+len(f.Body)) + binary.BigEndian.PutUint16(buf[0:2], Magic) + buf[2] = f.Version + binary.BigEndian.PutUint16(buf[3:5], f.Command) + binary.BigEndian.PutUint16(buf[5:7], f.Flags) + binary.BigEndian.PutUint64(buf[7:15], f.RequestID) + binary.BigEndian.PutUint32(buf[15:19], f.SequenceID) + binary.BigEndian.PutUint32(buf[19:23], uint32(len(f.Body))) + copy(buf[23:], f.Body) + return buf, nil +} + +func Decode(raw []byte, maxBody int) (Frame, error) { + if len(raw) < HeaderSize { + return Frame{}, errs.ErrInvalidFrame + } + magic := binary.BigEndian.Uint16(raw[0:2]) + if magic != Magic { + return Frame{}, errs.ErrInvalidFrame + } + bodyLen := int(binary.BigEndian.Uint32(raw[19:23])) + if bodyLen > maxBody || HeaderSize+bodyLen != len(raw) { + return Frame{}, errs.ErrFrameTooLarge + } + return Frame{ + Magic: magic, + Version: raw[2], + Command: binary.BigEndian.Uint16(raw[3:5]), + Flags: binary.BigEndian.Uint16(raw[5:7]), + RequestID: binary.BigEndian.Uint64(raw[7:15]), + SequenceID: binary.BigEndian.Uint32(raw[15:19]), + Body: append([]byte(nil), raw[23:]...), + }, nil +} + +// LooksLikeFrame reports whether buf starts with Magic. +func LooksLikeFrame(buf []byte) bool { + return len(buf) >= 2 && binary.BigEndian.Uint16(buf[0:2]) == Magic +} + +// MaybeFramePrefix is true when buf is a prefix of Magic (including a full Frame). +func MaybeFramePrefix(buf []byte) bool { + if len(buf) == 0 { + return true + } + if buf[0] != byte(Magic>>8) { + return false + } + if len(buf) == 1 { + return true + } + return binary.BigEndian.Uint16(buf[0:2]) == Magic +} + +// PeekBodyLength reads BodyLength from a complete header without copying the body. +func PeekBodyLength(header []byte) (int, error) { + if len(header) < HeaderSize { + return 0, errs.ErrInvalidFrame + } + if binary.BigEndian.Uint16(header[0:2]) != Magic { + return 0, errs.ErrInvalidFrame + } + return int(binary.BigEndian.Uint32(header[19:23])), nil +} diff --git a/internal/transport/tcp/protocol/frame_extra_test.go b/internal/transport/tcp/protocol/frame_extra_test.go new file mode 100644 index 0000000..8018115 --- /dev/null +++ b/internal/transport/tcp/protocol/frame_extra_test.go @@ -0,0 +1,88 @@ +package protocol + +import ( + "bytes" + "testing" + + "spark-server/internal/pkg/errs" +) + +func TestPeekBodyLengthAndPrefix(t *testing.T) { + raw, err := Encode(Frame{Version: CurrentVersion, Command: CmdHeartbeat, Body: []byte("abc")}) + if err != nil { + t.Fatal(err) + } + n, err := PeekBodyLength(raw[:HeaderSize]) + if err != nil || n != 3 { + t.Fatalf("peek body %d %v", n, err) + } + if !LooksLikeFrame(raw) { + t.Fatal("looks like frame") + } + if !MaybeFramePrefix(raw[:1]) { + t.Fatal("single magic byte is a frame prefix") + } + if MaybeFramePrefix([]byte("P")) { + t.Fatal("P is not a frame prefix") + } +} + +func TestCompressRoundTrip(t *testing.T) { + body := bytes.Repeat([]byte("spark-im-compress"), 40) + out, flags, err := PrepareBody(body, 32) + if err != nil { + t.Fatal(err) + } + if flags&FlagCompress == 0 { + t.Fatal("expected compress flag") + } + got, err := DecodeBody(out, flags, 65536) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, body) { + t.Fatal("decompress mismatch") + } +} + +func TestParsePrefixV1AndV2(t *testing.T) { + line := []byte("PROXY TCP4 203.0.113.10 192.0.2.1 443 9000\r\n") + h, n, err := ParsePrefix(append(append([]byte{}, line...), 'S', 'P')) + if err != nil { + t.Fatal(err) + } + if h.SrcIP.String() != "203.0.113.10" || h.SrcPort != 443 || n != len(line) { + t.Fatalf("v1 %+v n=%d want %d", h, n, len(line)) + } + if _, _, err := ParsePrefix([]byte("P")); !NeedMore(err) { + t.Fatalf("partial v1: %v", err) + } + v2 := buildProxyV2(t) + h2, n2, err := ParsePrefix(v2) + if err != nil { + t.Fatal(err) + } + if h2.SrcIP.String() != "203.0.113.10" || h2.SrcPort != 443 || n2 != len(v2) { + t.Fatalf("v2 %+v n=%d", h2, n2) + } + if _, _, err := ParsePrefix([]byte("GET /")); err != errs.ErrProxyHeader { + t.Fatalf("garbage: %v", err) + } +} + +func buildProxyV2(t *testing.T) []byte { + t.Helper() + buf := make([]byte, 16+12) + copy(buf, proxyV2Sig) + buf[12] = 0x21 + buf[13] = 0x11 + buf[14] = 0 + buf[15] = 12 + copy(buf[16:20], []byte{203, 0, 113, 10}) + copy(buf[20:24], []byte{192, 0, 2, 1}) + buf[24] = 1 + buf[25] = 187 // 443 + buf[26] = 0x23 + buf[27] = 0x28 // 9000 + return buf +} diff --git a/internal/transport/tcp/protocol/frame_test.go b/internal/transport/tcp/protocol/frame_test.go new file mode 100644 index 0000000..fd6d4ff --- /dev/null +++ b/internal/transport/tcp/protocol/frame_test.go @@ -0,0 +1,26 @@ +package protocol + +import "testing" + +func TestEncodeDecodeRoundTrip(t *testing.T) { + in := Frame{Version: 1, Command: CmdSendMessage, Flags: FlagCompress, RequestID: 9, SequenceID: 3, Body: []byte("hi")} + raw, err := Encode(in) + if err != nil { + t.Fatal(err) + } + out, err := Decode(raw, 65536) + if err != nil { + t.Fatal(err) + } + if out.Command != in.Command || out.RequestID != in.RequestID || string(out.Body) != "hi" { + t.Fatalf("round trip mismatch: %+v", out) + } +} + +func TestDecodeRejectsBadMagic(t *testing.T) { + raw, _ := Encode(Frame{Body: []byte("x")}) + raw[0] = 0 + if _, err := Decode(raw, 65536); err == nil { + t.Fatal("expected invalid frame") + } +} diff --git a/internal/transport/tcp/protocol/payload.go b/internal/transport/tcp/protocol/payload.go new file mode 100644 index 0000000..46244cf --- /dev/null +++ b/internal/transport/tcp/protocol/payload.go @@ -0,0 +1,146 @@ +package protocol + +import ( + "spark-server/internal/pkg/errs" + + "google.golang.org/protobuf/encoding/protowire" +) + +type AuthRequest struct { + Token string + DeviceID uint64 + AppVersion string + Language string +} + +type AuthResponse struct { + UserID int64 + DeviceID uint64 +} + +type ErrorBody struct { + Code int32 + Message string +} + +func EncodeAuthRequest(a AuthRequest) []byte { + out := appendStringField(nil, 1, a.Token) + out = appendVarintField(out, 2, a.DeviceID) + out = appendStringField(out, 3, a.AppVersion) + out = appendStringField(out, 4, a.Language) + return out +} + +func DecodeAuthRequest(body []byte) (AuthRequest, error) { + var req AuthRequest + for len(body) > 0 { + num, typ, tagLen := protowire.ConsumeTag(body) + if tagLen < 0 { + return AuthRequest{}, errs.ErrInvalidFrame + } + body = body[tagLen:] + val, n, err := consumeField(typ, body) + if err != nil { + return AuthRequest{}, err + } + body = body[n:] + assignAuth(&req, num, typ, val) + } + return req, nil +} + +func assignAuth(req *AuthRequest, num protowire.Number, typ protowire.Type, val []byte) { + switch { + case num == 1 && typ == protowire.BytesType: + req.Token = string(val) + case num == 2 && typ == protowire.VarintType: + v, _ := protowire.ConsumeVarint(val) + req.DeviceID = v + case num == 3 && typ == protowire.BytesType: + req.AppVersion = string(val) + case num == 4 && typ == protowire.BytesType: + req.Language = string(val) + } +} + +func EncodeAuthResponse(a AuthResponse) []byte { + out := appendVarintField(nil, 1, uint64(a.UserID)) + return appendVarintField(out, 2, a.DeviceID) +} + +func EncodeError(e ErrorBody) []byte { + out := appendVarintField(nil, 1, uint64(e.Code)) + return appendStringField(out, 2, e.Message) +} + +// DecodeError reads a protobuf-wire error payload. +func DecodeError(body []byte) (ErrorBody, error) { + var out ErrorBody + for len(body) > 0 { + num, typ, tagLen := protowire.ConsumeTag(body) + if tagLen < 0 { + return ErrorBody{}, errs.ErrInvalidFrame + } + body = body[tagLen:] + val, n, err := consumeField(typ, body) + if err != nil { + return ErrorBody{}, err + } + body = body[n:] + if num == 1 && typ == protowire.VarintType { + v, _ := protowire.ConsumeVarint(val) + out.Code = int32(v) + } + if num == 2 && typ == protowire.BytesType { + out.Message = string(val) + } + } + return out, nil +} + +func EncodeMsgID(id int64) []byte { + return appendVarintField(nil, 1, uint64(id)) +} + +func EncodeKick(reason string) []byte { + return appendStringField(nil, 1, reason) +} + +func appendStringField(out []byte, field protowire.Number, s string) []byte { + if s == "" { + return out + } + out = protowire.AppendVarint(out, protowire.EncodeTag(field, protowire.BytesType)) + return protowire.AppendString(out, s) +} + +func appendVarintField(out []byte, field protowire.Number, v uint64) []byte { + if v == 0 { + return out + } + out = protowire.AppendVarint(out, protowire.EncodeTag(field, protowire.VarintType)) + return protowire.AppendVarint(out, v) +} + +func consumeField(typ protowire.Type, body []byte) ([]byte, int, error) { + switch typ { + case protowire.VarintType: + v, n := protowire.ConsumeVarint(body) + if n < 0 { + return nil, 0, errs.ErrInvalidFrame + } + return protowire.AppendVarint(nil, v), n, nil + case protowire.BytesType: + v, n := protowire.ConsumeBytes(body) + if n < 0 { + return nil, 0, errs.ErrInvalidFrame + } + return append([]byte(nil), v...), n, nil + default: + n := protowire.ConsumeFieldValue(1, typ, body) + if n < 0 { + return nil, 0, errs.ErrInvalidFrame + } + return nil, n, nil + } +} diff --git a/internal/transport/tcp/protocol/proxy.go b/internal/transport/tcp/protocol/proxy.go new file mode 100644 index 0000000..210ad22 --- /dev/null +++ b/internal/transport/tcp/protocol/proxy.go @@ -0,0 +1,180 @@ +package protocol + +import ( + "bufio" + "encoding/binary" + "errors" + "io" + "net" + "strings" + + "spark-server/internal/pkg/errs" +) + +const maxProxyLine = 108 + +var ( + errNeedMore = errors.New("proxy header incomplete") + proxyV2Sig = []byte{0x0D, 0x0A, 0x0D, 0x0A, 0x00, 0x0D, 0x0A, 0x51, 0x55, 0x49, 0x54, 0x0A} + proxyV1Magic = []byte("PROXY ") +) + +type Header struct { + SrcIP net.IP + SrcPort int +} + +func NeedMore(err error) bool { + return errors.Is(err, errNeedMore) +} + +// ParsePrefix consumes a PROXY v1/v2 header from buf. n is bytes to discard. +func ParsePrefix(buf []byte) (h Header, n int, err error) { + if len(buf) == 0 { + return Header{}, 0, errNeedMore + } + if isPartialPrefix(buf, proxyV1Magic) || isPartialPrefix(buf, proxyV2Sig) { + return Header{}, 0, errNeedMore + } + if hasPrefix(buf, proxyV1Magic) { + return parseV1(buf) + } + if hasPrefix(buf, proxyV2Sig) { + return parseV2(buf) + } + return Header{}, 0, errs.ErrProxyHeader +} + +func isPartialPrefix(buf, magic []byte) bool { + if len(buf) >= len(magic) { + return false + } + return hasPrefix(magic, buf) +} + +func parseV1(buf []byte) (Header, int, error) { + idx := indexCRLF(buf) + if idx < 0 { + if len(buf) > maxProxyLine { + return Header{}, 0, errs.ErrProxyHeader + } + return Header{}, 0, errNeedMore + } + line := string(buf[:idx]) + h, err := parseV1Line(line) + if err != nil { + return Header{}, 0, err + } + return h, idx + 2, nil +} + +func parseV1Line(line string) (Header, error) { + fields := strings.Fields(strings.TrimSpace(line)) + if len(fields) < 2 || fields[0] != "PROXY" { + return Header{}, errs.ErrProxyHeader + } + if fields[1] == "UNKNOWN" { + return Header{}, nil + } + if len(fields) < 6 { + return Header{}, errs.ErrProxyHeader + } + ip := net.ParseIP(fields[2]) + if ip == nil { + return Header{}, errs.ErrProxyHeader + } + port, ok := atoi(fields[4]) + if !ok { + return Header{}, errs.ErrProxyHeader + } + return Header{SrcIP: ip, SrcPort: port}, nil +} + +func parseV2(buf []byte) (Header, int, error) { + if len(buf) < 16 { + return Header{}, 0, errNeedMore + } + addrLen := int(binary.BigEndian.Uint16(buf[14:16])) + total := 16 + addrLen + if len(buf) < total { + return Header{}, 0, errNeedMore + } + h := Header{} + fam := buf[13] >> 4 + if fam == 1 && addrLen >= 12 { // IPv4 + h.SrcIP = net.IPv4(buf[16], buf[17], buf[18], buf[19]) + h.SrcPort = int(binary.BigEndian.Uint16(buf[24:26])) + } + if fam == 2 && addrLen >= 36 { // IPv6 + h.SrcIP = net.IP(append([]byte(nil), buf[16:32]...)) + h.SrcPort = int(binary.BigEndian.Uint16(buf[48:50])) + } + return h, total, nil +} + +func ReadHeader(r *bufio.Reader) (Header, error) { + var acc []byte + for { + b, err := r.ReadByte() + if err != nil { + return Header{}, err + } + acc = append(acc, b) + if len(acc) > maxProxyLine+16 { + return Header{}, errs.ErrProxyHeader + } + h, n, err := ParsePrefix(acc) + if NeedMore(err) { + continue + } + if err != nil { + return Header{}, err + } + if n < len(acc) { + for i := len(acc) - 1; i >= n; i-- { + _ = r.UnreadByte() + } + } + return h, nil + } +} + +func Discard(r io.Reader) error { + _, err := io.Copy(io.Discard, r) + return err +} + +func hasPrefix(b, pre []byte) bool { + if len(b) < len(pre) { + return false + } + for i := range pre { + if b[i] != pre[i] { + return false + } + } + return true +} + +func indexCRLF(b []byte) int { + for i := 0; i+1 < len(b); i++ { + if b[i] == '\r' && b[i+1] == '\n' { + return i + } + } + return -1 +} + +func atoi(s string) (int, bool) { + n := 0 + if s == "" { + return 0, false + } + for _, c := range s { + if c < '0' || c > '9' { + return 0, false + } + n = n*10 + int(c-'0') + } + return n, true +} diff --git a/internal/transport/tcp/protocol/proxy_test.go b/internal/transport/tcp/protocol/proxy_test.go new file mode 100644 index 0000000..f054048 --- /dev/null +++ b/internal/transport/tcp/protocol/proxy_test.go @@ -0,0 +1,26 @@ +package protocol + +import ( + "bufio" + "net" + "strings" + "testing" +) + +func TestReadHeaderV1(t *testing.T) { + r := bufio.NewReader(strings.NewReader("PROXY TCP4 203.0.113.10 192.0.2.1 443 9000\r\n")) + h, err := ReadHeader(r) + if err != nil { + t.Fatal(err) + } + if !h.SrcIP.Equal(net.ParseIP("203.0.113.10")) || h.SrcPort != 443 { + t.Fatalf("unexpected header %+v", h) + } +} + +func TestReadHeaderRejectsGarbage(t *testing.T) { + r := bufio.NewReader(strings.NewReader("GET / HTTP/1.1\r\n")) + if _, err := ReadHeader(r); err == nil { + t.Fatal("expected error") + } +} diff --git a/internal/transport/tcp/server.go b/internal/transport/tcp/server.go new file mode 100644 index 0000000..c77abb9 --- /dev/null +++ b/internal/transport/tcp/server.go @@ -0,0 +1,252 @@ +package tcp + +import ( + "context" + "crypto/tls" + "log/slog" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/panjf2000/gnet/v2" + + "spark-server/internal/module/auth" + "spark-server/internal/transport/tcp/session" +) + +const ( + defaultMaxFrame = 65536 + defaultCompressMin = 512 + defaultIdleTimeout = 120 * time.Second + defaultOutbound = 1024 + defaultRatePerSec = 20 + defaultHandlerTimeout = 5 * time.Second + maxFramesPerEvent = 32 +) + +// Metrics is a narrow counter hook. EventLoop must not block on exporters. +type Metrics interface { + Inc(name string) +} + +type nopMetrics struct{} + +func (nopMetrics) Inc(string) {} + +// MsgIDGenerator is satisfied by infrastructure/id.Generator. +type MsgIDGenerator interface { + NextMsgID() (int64, error) +} + +// Server is the gnet event engine. OnTraffic must not call MySQL, Redis, or deliver fanout. +type Server struct { + gnet.BuiltinEventEngine + log *slog.Logger + pool TaskPool + sessions session.Manager + eng gnet.Engine + addr string + maxFrame int + opts []gnet.Option + proxyProtocol bool + idleTimeout time.Duration + outboundSize int + ratePerSec int + compressMin int + handlerTimeout time.Duration + auth auth.Service + ids MsgIDGenerator + router *Router + metrics Metrics + conns sync.Map + seq atomic.Uint64 + ready chan struct{} + readyOnce sync.Once +} + +// Options configure the TCP listener. TLS is terminated at Nginx in production. +type Options struct { + Addr string + Multicore bool + KeepAlive time.Duration + RecvBuffer int + SendBuffer int + MaxFrameSize int + TLS *tls.Config + ProxyProtocol bool + IdleTimeout time.Duration + OutboundSize int + RatePerSec int + CompressMin int + HandlerTimeout time.Duration + Auth auth.Service + IDs MsgIDGenerator + Metrics Metrics +} + +// NewServer builds a gnet IM listener. Call Start to bind. +func NewServer(log *slog.Logger, pool TaskPool, sessions session.Manager, o *Options) *Server { + if o == nil { + o = &Options{} + } + applyTCPDefaults(o) + s := &Server{ + log: log, + pool: pool, + sessions: sessions, + addr: o.Addr, + maxFrame: o.MaxFrameSize, + proxyProtocol: o.ProxyProtocol, + idleTimeout: o.IdleTimeout, + outboundSize: o.OutboundSize, + ratePerSec: o.RatePerSec, + compressMin: o.CompressMin, + handlerTimeout: o.HandlerTimeout, + auth: o.Auth, + ids: o.IDs, + metrics: o.Metrics, + router: NewRouter(), + ready: make(chan struct{}), + } + if s.metrics == nil { + s.metrics = nopMetrics{} + } + s.registerDefaultHandlers() + if o.TLS != nil { + log.Info("tls cert loaded; terminate at nginx or wrap listener (gnet v2 has no native TLS)") + } + s.opts = gnetOptions(o) + return s +} + +func applyTCPDefaults(o *Options) { + if o.MaxFrameSize <= 0 { + o.MaxFrameSize = defaultMaxFrame + } + if o.CompressMin <= 0 { + o.CompressMin = defaultCompressMin + } + if o.IdleTimeout <= 0 { + o.IdleTimeout = defaultIdleTimeout + } + if o.OutboundSize <= 0 { + o.OutboundSize = defaultOutbound + } + if o.RatePerSec <= 0 { + o.RatePerSec = defaultRatePerSec + } + if o.HandlerTimeout <= 0 { + o.HandlerTimeout = defaultHandlerTimeout + } +} + +func gnetOptions(o *Options) []gnet.Option { + opts := []gnet.Option{ + gnet.WithMulticore(o.Multicore), + gnet.WithReusePort(true), + gnet.WithTicker(true), + } + if o.KeepAlive > 0 { + opts = append(opts, gnet.WithTCPKeepAlive(o.KeepAlive)) + } + if o.RecvBuffer > 0 { + opts = append(opts, gnet.WithSocketRecvBuffer(o.RecvBuffer)) + } + if o.SendBuffer > 0 { + opts = append(opts, gnet.WithSocketSendBuffer(o.SendBuffer)) + } + return opts +} + +// Router returns the command table so modules can register SEND/SYNC later. +func (s *Server) Router() *Router { return s.router } + +func (s *Server) OnBoot(eng gnet.Engine) gnet.Action { + s.eng = eng + s.log.Info("tcp listening", "addr", s.addr) + s.readyOnce.Do(func() { close(s.ready) }) + return gnet.None +} + +func (s *Server) OnOpen(c gnet.Conn) ([]byte, gnet.Action) { + s.attach(c) + return nil, gnet.None +} + +func (s *Server) OnClose(c gnet.Conn, _ error) gnet.Action { + st := connFrom(c) + if st == nil { + return gnet.None + } + s.conns.Delete(st.sess.ID()) + _ = s.sessions.Unbind(st.sess.ID()) + st.sess.Detach() + return gnet.None +} + +func (s *Server) OnTick() (time.Duration, gnet.Action) { + s.conns.Range(func(_, v any) bool { + st, ok := v.(*connState) + if !ok { + return true + } + if st.sess.Idle(s.idleTimeout) { + _ = st.gc.Close() + return true + } + s.flush(st) + return true + }) + return time.Second, gnet.None +} + +func (s *Server) OnTraffic(c gnet.Conn) gnet.Action { + st := connFrom(c) + if st == nil { + return gnet.Close + } + if action := s.consumeProxy(st); action != gnet.None { + return action + } + if !st.sess.ProxyDone() { + s.flush(st) + return gnet.None + } + for i := 0; i < maxFramesPerEvent; i++ { + action, more := s.consumeFrame(st) + if action != gnet.None { + return action + } + if !more { + break + } + } + s.flush(st) + return gnet.None +} + +func (s *Server) Start() error { + return gnet.Run(s, "tcp://"+s.addr, s.opts...) +} + +func (s *Server) Stop(ctx context.Context) error { + return s.eng.Stop(ctx) +} + +func (s *Server) attach(c gnet.Conn) { + id := strconv.FormatUint(s.seq.Add(1), 10) + remote := "unknown" + if addr := c.RemoteAddr(); addr != nil { + remote = addr.String() + } + sess := session.NewConn(id, remote, s.outboundSize) + if !s.proxyProtocol { + sess.MarkProxySkipped() + } + st := &connState{sess: sess, gc: c, limit: newLimiter(s.ratePerSec)} + sess.SetIO(func(b []byte) error { return c.AsyncWrite(b, nil) }, func() error { return c.Close() }) + sess.SetWake(func() { _ = c.Wake(nil) }) + c.SetContext(st) + s.conns.Store(id, st) +} diff --git a/internal/transport/tcp/server_test.go b/internal/transport/tcp/server_test.go new file mode 100644 index 0000000..cdcc0d1 --- /dev/null +++ b/internal/transport/tcp/server_test.go @@ -0,0 +1,218 @@ +package tcp + +import ( + "context" + "encoding/binary" + "io" + "log/slog" + "net" + "testing" + "time" + + "spark-server/internal/infrastructure/id" + "spark-server/internal/module/auth" + "spark-server/internal/transport/tcp/protocol" + "spark-server/internal/transport/tcp/session" +) + +func TestHeartbeatWithoutAuth(t *testing.T) { + srv := startTCP(t, &Options{}) + c := dial(t, srv.addr) + defer c.Close() + writeFrame(t, c, protocol.Frame{Version: protocol.CurrentVersion, Command: protocol.CmdHeartbeat, SequenceID: 3}) + got := readFrame(t, c) + if got.Command != protocol.CmdHeartbeat || got.SequenceID != 3 { + t.Fatalf("heartbeat reply %+v", got) + } +} + +func TestAuthGateAndGenerateMsgID(t *testing.T) { + ids, err := id.New(1) + if err != nil { + t.Fatal(err) + } + srv := startTCP(t, &Options{ + Auth: auth.Static{Token: "secret", UserID: 42, DeviceID: 7}, + IDs: ids, + }) + c := dial(t, srv.addr) + defer c.Close() + + writeFrame(t, c, protocol.Frame{ + Version: protocol.CurrentVersion, Command: protocol.CmdGenerateMsgID, RequestID: 9, SequenceID: 1, + }) + denied := readFrame(t, c) + eb, err := protocol.DecodeError(denied.Body) + if err != nil { + t.Fatal(err) + } + if eb.Code != protocol.CodeNotAuth { + t.Fatalf("want not-auth, got %+v", eb) + } + + writeFrame(t, c, protocol.Frame{ + Version: protocol.CurrentVersion, Command: protocol.CmdAuth, RequestID: 2, SequenceID: 2, + Body: protocol.EncodeAuthRequest(protocol.AuthRequest{Token: "secret", DeviceID: 7}), + }) + authed := readFrame(t, c) + if authed.Command != protocol.CmdAuth || len(authed.Body) == 0 { + t.Fatalf("auth reply %+v", authed) + } + + writeFrame(t, c, protocol.Frame{ + Version: protocol.CurrentVersion, Command: protocol.CmdGenerateMsgID, RequestID: 3, SequenceID: 3, + }) + msgid := readFrame(t, c) + if msgid.Command != protocol.CmdGenerateMsgID || len(msgid.Body) == 0 { + t.Fatalf("msgid reply %+v", msgid) + } +} + +func TestProxyProtocolThenHeartbeat(t *testing.T) { + ids, err := id.New(1) + if err != nil { + t.Fatal(err) + } + srv := startTCP(t, &Options{ + ProxyProtocol: true, + Auth: auth.Static{Token: "secret", UserID: 42, DeviceID: 7}, + IDs: ids, + }) + c := dial(t, srv.addr) + defer c.Close() + if _, err := c.Write([]byte("PROXY TCP4 203.0.113.10 192.0.2.1 443 9000\r\n")); err != nil { + t.Fatal(err) + } + writeFrame(t, c, protocol.Frame{ + Version: protocol.CurrentVersion, Command: protocol.CmdAuth, SequenceID: 1, + Body: protocol.EncodeAuthRequest(protocol.AuthRequest{Token: "secret", DeviceID: 7}), + }) + authed := readFrame(t, c) + if authed.Command != protocol.CmdAuth { + t.Fatalf("auth after proxy %+v", authed) + } + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + sessions := srv.sessions.GetUserSessions(42) + if len(sessions) == 1 && sessions[0].RealIP() == "203.0.113.10:443" { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("real ip not set, sessions=%v", dumpRealIP(srv.sessions.GetUserSessions(42))) +} + +func dumpRealIP(ss []session.Session) []string { + out := make([]string, 0, len(ss)) + for _, s := range ss { + out = append(out, s.RealIP()) + } + return out +} + +func startTCP(t *testing.T, o *Options) *Server { + t.Helper() + if o == nil { + o = &Options{} + } + if o.Addr == "" { + o.Addr = freeAddr(t) + } + if o.IdleTimeout == 0 { + o.IdleTimeout = time.Hour + } + o.KeepAlive = 30 * time.Second + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + pool := NewTaskPool(4, 32) + t.Cleanup(pool.Close) + s := NewServer(log, pool, session.NewManager(), o) + errCh := make(chan error, 1) + go func() { errCh <- s.Start() }() + select { + case <-s.ready: + case err := <-errCh: + t.Fatalf("start: %v", err) + case <-time.After(3 * time.Second): + t.Fatal("boot timeout") + } + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + _ = s.Stop(ctx) + select { + case <-errCh: + case <-time.After(3 * time.Second): + } + }) + return s +} + +func freeAddr(t *testing.T) string { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + addr := ln.Addr().String() + _ = ln.Close() + return addr +} + +func dial(t *testing.T, addr string) net.Conn { + t.Helper() + var last error + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + c, err := net.DialTimeout("tcp", addr, 200*time.Millisecond) + if err == nil { + _ = c.SetDeadline(time.Now().Add(3 * time.Second)) + return c + } + last = err + time.Sleep(20 * time.Millisecond) + } + t.Fatalf("dial %s: %v", addr, last) + return nil +} + +func writeFrame(t *testing.T, c net.Conn, f protocol.Frame) { + t.Helper() + raw, err := protocol.Encode(f) + if err != nil { + t.Fatal(err) + } + if _, err := c.Write(raw); err != nil { + t.Fatal(err) + } +} + +func readFrame(t *testing.T, c net.Conn) protocol.Frame { + t.Helper() + header := make([]byte, protocol.HeaderSize) + if _, err := io.ReadFull(c, header); err != nil { + t.Fatal(err) + } + bodyLen := int(binary.BigEndian.Uint32(header[19:23])) + raw := make([]byte, protocol.HeaderSize+bodyLen) + copy(raw, header) + if bodyLen > 0 { + if _, err := io.ReadFull(c, raw[protocol.HeaderSize:]); err != nil { + t.Fatal(err) + } + } + f, err := protocol.Decode(raw, 65536) + if err != nil { + t.Fatal(err) + } + return f +} + +func TestLimiterAllow(t *testing.T) { + l := newLimiter(1) + if !l.Allow() { + t.Fatal("first token") + } + if l.Allow() { + t.Fatal("burst exhausted") + } +} diff --git a/internal/transport/tcp/session/conn.go b/internal/transport/tcp/session/conn.go new file mode 100644 index 0000000..6e7be06 --- /dev/null +++ b/internal/transport/tcp/session/conn.go @@ -0,0 +1,202 @@ +package session + +import ( + "sync/atomic" + "time" + + "spark-server/internal/pkg/errs" + "spark-server/internal/pkg/reqctx" + "spark-server/internal/transport/tcp/protocol" +) + +const ( + StateConnecting uint32 = iota + StateConnected + StateAuthenticated + StateActive + StateClosing + StateClosed +) + +type Conn struct { + id string + userID atomic.Int64 + deviceID atomic.Uint64 + realIP atomic.Value // string + remote string + state atomic.Uint32 + lastActive atomic.Int64 + drops atomic.Uint64 + proxyDone atomic.Bool + closed atomic.Bool + + outbound chan *Packet + write func([]byte) error + closeFn func() error + wake func() +} + +func NewConn(id, remote string, outboundSize int) *Conn { + if outboundSize < 1 { + outboundSize = 1024 + } + c := &Conn{ + id: id, + remote: remote, + outbound: make(chan *Packet, outboundSize), + } + c.state.Store(StateConnected) + c.lastActive.Store(time.Now().UnixNano()) + c.realIP.Store("") + return c +} + +func (c *Conn) SetIO(write func([]byte) error, closeFn func() error) { + c.write = write + c.closeFn = closeFn +} + +// SetWake registers EventLoop wakeup after outbound enqueue. +func (c *Conn) SetWake(wake func()) { + c.wake = wake +} + +func (c *Conn) ID() string { return c.id } +func (c *Conn) UserID() int64 { return c.userID.Load() } +func (c *Conn) DeviceID() uint64 { return c.deviceID.Load() } +func (c *Conn) RemoteAddr() string { return c.remote } +func (c *Conn) Drops() uint64 { return c.drops.Load() } + +func (c *Conn) RealIP() string { + v, _ := c.realIP.Load().(string) + if v != "" { + return v + } + return c.remote +} + +func (c *Conn) SetRealIP(ip string) { + c.realIP.Store(ip) + c.proxyDone.Store(true) +} + +func (c *Conn) MarkProxySkipped() { c.proxyDone.Store(true) } +func (c *Conn) ProxyDone() bool { return c.proxyDone.Load() } + +func (c *Conn) Touch() { c.lastActive.Store(time.Now().UnixNano()) } + +func (c *Conn) Idle(timeout time.Duration) bool { + if timeout <= 0 { + return false + } + return time.Since(time.Unix(0, c.lastActive.Load())) > timeout +} + +func (c *Conn) State() uint32 { return c.state.Load() } + +func (c *Conn) Authenticated() bool { + st := c.state.Load() + return st == StateAuthenticated || st == StateActive +} + +func (c *Conn) Authenticate(userID int64, deviceID uint64) { + c.userID.Store(userID) + c.deviceID.Store(deviceID) + c.state.Store(StateAuthenticated) +} + +func (c *Conn) Activate() { + if c.state.Load() == StateAuthenticated { + c.state.Store(StateActive) + } +} + +func (c *Conn) Send(packet *Packet) error { + if c.closed.Load() { + return errs.ErrClosed + } + select { + case c.outbound <- packet: + c.notifyWake() + return nil + default: + c.drops.Add(1) + return errs.ErrOutboundDropped + } +} + +func (c *Conn) notifyWake() { + if c.wake != nil { + c.wake() + } +} + +func (c *Conn) Close() error { + if !c.closed.CompareAndSwap(false, true) { + return nil + } + c.state.Store(StateClosing) + c.flushDirect() + c.state.Store(StateClosed) + if c.closeFn != nil { + return c.closeFn() + } + return nil +} + +// Detach marks the session closed without touching the socket (OnClose). +func (c *Conn) Detach() { + c.closed.Store(true) + c.state.Store(StateClosed) + c.write = nil + c.closeFn = nil + c.wake = nil +} + +func (c *Conn) flushDirect() { + for _, p := range c.DrainOutbound(64) { + c.writePacket(p) + } +} + +func (c *Conn) writePacket(p *Packet) { + if c.write == nil || p == nil { + return + } + raw, err := protocol.Encode(protocol.Frame{ + Magic: protocol.Magic, Version: protocol.CurrentVersion, + Command: p.Command, Flags: p.Flags, RequestID: p.RequestID, SequenceID: p.SequenceID, Body: p.Body, + }) + if err != nil { + return + } + _ = c.write(raw) +} + +func (c *Conn) DrainOutbound(limit int) []*Packet { + if limit < 1 { + limit = 32 + } + out := make([]*Packet, 0, limit) + for len(out) < limit { + select { + case p := <-c.outbound: + if p != nil { + out = append(out, p) + } + default: + return out + } + } + return out +} + +func (c *Conn) RequestContext() *reqctx.RequestContext { + return &reqctx.RequestContext{ + UserID: c.UserID(), + DeviceID: c.DeviceID(), + ConnectionID: c.id, + RemoteAddr: c.remote, + RealIP: c.RealIP(), + } +} diff --git a/internal/transport/tcp/session/conn_test.go b/internal/transport/tcp/session/conn_test.go new file mode 100644 index 0000000..1d18cae --- /dev/null +++ b/internal/transport/tcp/session/conn_test.go @@ -0,0 +1,55 @@ +package session + +import ( + "testing" + + "spark-server/internal/pkg/errs" + "spark-server/internal/transport/tcp/protocol" +) + +func TestOutboundDropDoesNotClose(t *testing.T) { + c := NewConn("1", "127.0.0.1:1", 1) + if err := c.Send(&Packet{Command: protocol.CmdHeartbeat}); err != nil { + t.Fatal(err) + } + err := c.Send(&Packet{Command: protocol.CmdHeartbeat}) + if err != errs.ErrOutboundDropped { + t.Fatalf("want drop, got %v", err) + } + if c.Drops() != 1 { + t.Fatalf("drops %d", c.Drops()) + } + if c.State() == StateClosed { + t.Fatal("drop must not close") + } +} + +func TestKickSameDevice(t *testing.T) { + m := NewManager() + var kicked []byte + old := NewConn("old", "10.0.0.1:1", 8) + old.Authenticate(7, 9) + old.SetIO(func(b []byte) error { kicked = append(kicked, b...); return nil }, nil) + if err := m.Bind(old); err != nil { + t.Fatal(err) + } + n := NewConn("new", "10.0.0.1:2", 8) + n.Authenticate(7, 9) + if err := m.Bind(n); err != nil { + t.Fatal(err) + } + if old.State() != StateClosed { + t.Fatalf("old state %d", old.State()) + } + got, err := protocol.Decode(kicked, 65536) + if err != nil { + t.Fatal(err) + } + if got.Command != protocol.CmdKick { + t.Fatalf("want kick, got %d", got.Command) + } + sessions := m.GetUserSessions(7) + if len(sessions) != 1 || sessions[0].ID() != "new" { + t.Fatalf("sessions %+v", sessions) + } +} diff --git a/internal/transport/tcp/session/kick.go b/internal/transport/tcp/session/kick.go new file mode 100644 index 0000000..a8b8929 --- /dev/null +++ b/internal/transport/tcp/session/kick.go @@ -0,0 +1,31 @@ +package session + +import ( + "spark-server/internal/transport/tcp/protocol" +) + +func (m *memManager) Get(sessionID string) (Session, bool) { + m.mu.RLock() + defer m.mu.RUnlock() + s, ok := m.byID[sessionID] + return s, ok +} + +func (m *memManager) kickSameDevice(incoming Session) { + userMap := m.byUser[incoming.UserID()] + if userMap == nil { + return + } + for _, existing := range userMap { + if existing.DeviceID() != incoming.DeviceID() || existing.ID() == incoming.ID() { + continue + } + _ = existing.Send(&Packet{ + Command: protocol.CmdKick, + Body: protocol.EncodeKick("replaced by new connection"), + }) + _ = existing.Close() + delete(m.byID, existing.ID()) + delete(userMap, existing.ID()) + } +} diff --git a/internal/transport/tcp/session/manager.go b/internal/transport/tcp/session/manager.go new file mode 100644 index 0000000..5a9803e --- /dev/null +++ b/internal/transport/tcp/session/manager.go @@ -0,0 +1,95 @@ +// Package session keeps TCP connections in process memory. +package session + +import ( + "context" + "sync" +) + +type Packet struct { + Command uint16 + Flags uint16 + RequestID uint64 + SequenceID uint32 + Body []byte +} + +type Session interface { + ID() string + UserID() int64 + DeviceID() uint64 + RealIP() string + Send(packet *Packet) error + Close() error +} + +type Manager interface { + Bind(session Session) error + Unbind(sessionID string) error + Get(sessionID string) (Session, bool) + GetUserSessions(userID int64) []Session + SendToUserExceptDevice(ctx context.Context, userID int64, exceptDeviceID uint64, packet *Packet) error +} + +type memManager struct { + mu sync.RWMutex + byID map[string]Session + byUser map[int64]map[string]Session +} + +func NewManager() Manager { + return &memManager{ + byID: make(map[string]Session), + byUser: make(map[int64]map[string]Session), + } +} + +func (m *memManager) Bind(s Session) error { + m.mu.Lock() + defer m.mu.Unlock() + if old, ok := m.byID[s.ID()]; ok { + _ = old.Close() + } + m.kickSameDevice(s) + if m.byUser[s.UserID()] == nil { + m.byUser[s.UserID()] = make(map[string]Session) + } + m.byID[s.ID()] = s + m.byUser[s.UserID()][s.ID()] = s + return nil +} + +func (m *memManager) Unbind(sessionID string) error { + m.mu.Lock() + defer m.mu.Unlock() + s, ok := m.byID[sessionID] + if !ok { + return nil + } + delete(m.byID, sessionID) + if um := m.byUser[s.UserID()]; um != nil { + delete(um, sessionID) + } + return nil +} + +func (m *memManager) GetUserSessions(userID int64) []Session { + m.mu.RLock() + defer m.mu.RUnlock() + um := m.byUser[userID] + out := make([]Session, 0, len(um)) + for _, s := range um { + out = append(out, s) + } + return out +} + +func (m *memManager) SendToUserExceptDevice(_ context.Context, userID int64, exceptDeviceID uint64, packet *Packet) error { + for _, s := range m.GetUserSessions(userID) { + if s.DeviceID() == exceptDeviceID { + continue + } + _ = s.Send(packet) + } + return nil +} diff --git a/internal/transport/tcp/taskpool.go b/internal/transport/tcp/taskpool.go new file mode 100644 index 0000000..56f05c3 --- /dev/null +++ b/internal/transport/tcp/taskpool.go @@ -0,0 +1,66 @@ +package tcp + +import ( + "context" + "sync" + + "spark-server/internal/pkg/errs" +) + +// TaskPool runs inbound business (MySQL included). EventLoop must only Submit. +type TaskPool interface { + Submit(ctx context.Context, task func()) error + Close() +} + +type pool struct { + tasks chan func() + workers int + wg sync.WaitGroup + once sync.Once +} + +func NewTaskPool(workers, queue int) TaskPool { + if workers < 1 { + workers = 1 + } + if queue < 1 { + queue = 1 + } + p := &pool{tasks: make(chan func(), queue), workers: workers} + for i := 0; i < workers; i++ { + p.wg.Add(1) + go p.loop() + } + return p +} + +func (p *pool) loop() { + defer p.wg.Done() + for task := range p.tasks { + if task != nil { + task() + } + } +} + +func (p *pool) Submit(ctx context.Context, task func()) error { + if task == nil { + return errs.ErrInvalidArgument + } + select { + case <-ctx.Done(): + return ctx.Err() + case p.tasks <- task: + return nil + default: + return errs.ErrBusy + } +} + +func (p *pool) Close() { + p.once.Do(func() { + close(p.tasks) + p.wg.Wait() + }) +} diff --git a/internal/transport/tcp/taskpool_test.go b/internal/transport/tcp/taskpool_test.go new file mode 100644 index 0000000..72eaa52 --- /dev/null +++ b/internal/transport/tcp/taskpool_test.go @@ -0,0 +1,43 @@ +package tcp + +import ( + "context" + "testing" + "time" + + "spark-server/internal/pkg/errs" +) + +func TestTaskPoolRunsJob(t *testing.T) { + p := NewTaskPool(1, 4) + defer p.Close() + done := make(chan struct{}) + err := p.Submit(context.Background(), func() { close(done) }) + if err != nil { + t.Fatal(err) + } + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("timeout") + } +} + +func TestTaskPoolBusy(t *testing.T) { + p := NewTaskPool(1, 1) + defer p.Close() + started := make(chan struct{}) + block := make(chan struct{}) + if err := p.Submit(context.Background(), func() { close(started); <-block }); err != nil { + t.Fatal(err) + } + <-started + if err := p.Submit(context.Background(), func() {}); err != nil { + t.Fatal(err) + } + err := p.Submit(context.Background(), func() {}) + close(block) + if err != errs.ErrBusy { + t.Fatalf("want busy, got %v", err) + } +} diff --git a/migrations/0001_init.sql b/migrations/0001_init.sql new file mode 100644 index 0000000..c228095 --- /dev/null +++ b/migrations/0001_init.sql @@ -0,0 +1,62 @@ +-- Spark IM V3 initial schema. Message body lives in MySQL; dialog list lives in Redis. + +CREATE TABLE IF NOT EXISTS conversations ( + id BIGINT PRIMARY KEY, + type TINYINT NOT NULL, + last_msg_id BIGINT NOT NULL DEFAULT 0, + last_msg_at DATETIME(3) NOT NULL, + created_at DATETIME(3) NOT NULL +); + +CREATE TABLE IF NOT EXISTS p2p_conversations ( + user_small BIGINT NOT NULL, + user_big BIGINT NOT NULL, + conversation_id BIGINT NOT NULL, + PRIMARY KEY (user_small, user_big), + UNIQUE KEY uk_conversation (conversation_id) +); + +CREATE TABLE IF NOT EXISTS conversation_members ( + conversation_id BIGINT NOT NULL, + user_id BIGINT NOT NULL, + last_read_msg_id BIGINT NOT NULL DEFAULT 0, + joined_at DATETIME(3) NOT NULL, + muted TINYINT NOT NULL DEFAULT 0, + PRIMARY KEY (conversation_id, user_id), + KEY idx_user_id (user_id) +); + +CREATE TABLE IF NOT EXISTS messages ( + id BIGINT PRIMARY KEY, + conversation_id BIGINT NOT NULL, + sender_id BIGINT NOT NULL, + client_msg_id VARCHAR(64) NOT NULL, + message_type SMALLINT NOT NULL, + content VARBINARY(65535) NOT NULL, + status TINYINT NOT NULL DEFAULT 1, + created_at DATETIME(3) NOT NULL, + UNIQUE KEY uk_sender_conv_client_msg (sender_id, conversation_id, client_msg_id), + KEY idx_conversation_id (conversation_id, id) +); + +CREATE TABLE IF NOT EXISTS mailbox_events ( + id BIGINT PRIMARY KEY, + user_id BIGINT NOT NULL, + device_id BIGINT NOT NULL, + seq BIGINT NOT NULL, + event_type SMALLINT NOT NULL, + conversation_id BIGINT NULL, + payload VARBINARY(4096), + created_at DATETIME(3) NOT NULL, + UNIQUE KEY uk_user_device_seq (user_id, device_id, seq), + KEY idx_user_device (user_id, device_id, seq) +); + +CREATE TABLE IF NOT EXISTS device_sync_cursors ( + user_id BIGINT NOT NULL, + device_id BIGINT NOT NULL, + mailbox_seq BIGINT NOT NULL DEFAULT 0, + dialog_score BIGINT NOT NULL DEFAULT 0, + updated_at DATETIME(3) NOT NULL, + PRIMARY KEY (user_id, device_id) +);