package logic
import (
"context"
"encoding/xml"
"fmt"
"html"
"io"
"net/http"
"net/url"
"regexp"
"strings"
"time"
"unicode/utf8"
)
const (
rssMaxBody = 2 << 20
rssTimeout = 20 * time.Second
summaryRunes = 280
)
// FeedItem is one public RSS/Atom entry (title + summary + link).
type FeedItem struct {
Title string
Summary string
Link string
Source string
ImageURL string // optional enclosure / media:content / first
}
type rssFeed struct {
Channel struct {
Title string `xml:"title"`
Items []rssItem `xml:"item"`
} `xml:"channel"`
}
type rssItem struct {
Title string `xml:"title"`
Link string `xml:"link"`
Description string `xml:"description"`
GUID string `xml:"guid"`
Enclosure *rssEnclosure `xml:"enclosure"`
MediaContent []rssMediaContent `xml:"http://search.yahoo.com/mrss/ content"`
MediaThumb []rssMediaContent `xml:"http://search.yahoo.com/mrss/ thumbnail"`
}
type rssEnclosure struct {
URL string `xml:"url,attr"`
Type string `xml:"type,attr"`
}
type rssMediaContent struct {
URL string `xml:"url,attr"`
Type string `xml:"type,attr"`
Medium string `xml:"medium,attr"`
}
type atomFeed struct {
Title string `xml:"title"`
Entries []atomEntry `xml:"entry"`
}
type atomEntry struct {
Title string `xml:"title"`
Summary string `xml:"summary"`
Content string `xml:"content"`
Links []atomLink `xml:"link"`
ID string `xml:"id"`
}
type atomLink struct {
Href string `xml:"href,attr"`
Rel string `xml:"rel,attr"`
Type string `xml:"type,attr"`
}
var tagRe = regexp.MustCompile(`(?s)<[^>]*>`)
var imgSrcRe = regexp.MustCompile(`(?i)
]+src=["']([^"']+)["']`)
// FetchFeed loads and parses a public RSS 2.0 or Atom URL.
func FetchFeed(ctx context.Context, client *http.Client, feedURL string) ([]FeedItem, error) {
feedURL = strings.TrimSpace(feedURL)
if feedURL == "" {
return nil, fmt.Errorf("empty feed url")
}
if client == nil {
client = &http.Client{Timeout: rssTimeout}
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, feedURL, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", "SparkIM-SquareBot/1.0 (+https://spark.wyrs365.com)")
req.Header.Set("Accept", "application/rss+xml, application/atom+xml, application/xml, text/xml, */*")
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return nil, fmt.Errorf("feed status %d", resp.StatusCode)
}
body, err := io.ReadAll(io.LimitReader(resp.Body, rssMaxBody))
if err != nil {
return nil, err
}
return parseFeed(body, feedURL)
}
func parseFeed(body []byte, feedURL string) ([]FeedItem, error) {
var rss rssFeed
if err := xml.Unmarshal(body, &rss); err == nil && len(rss.Channel.Items) > 0 {
src := strings.TrimSpace(rss.Channel.Title)
out := make([]FeedItem, 0, len(rss.Channel.Items))
for i := range rss.Channel.Items {
it := &rss.Channel.Items[i]
link := firstNonEmpty(strings.TrimSpace(it.Link), strings.TrimSpace(it.GUID))
item := FeedItem{
Title: cleanText(it.Title),
Summary: cleanText(it.Description),
Link: normalizeURL(link),
Source: src,
ImageURL: rssImageURL(it),
}
if item.Title == "" || item.Link == "" {
continue
}
out = append(out, item)
}
if len(out) > 0 {
return out, nil
}
}
var atom atomFeed
if err := xml.Unmarshal(body, &atom); err != nil {
return nil, fmt.Errorf("parse feed %s: %w", feedURL, err)
}
src := cleanText(atom.Title)
out := make([]FeedItem, 0, len(atom.Entries))
for i := range atom.Entries {
e := &atom.Entries[i]
link := atomBestLink(e.Links)
if link == "" {
link = strings.TrimSpace(e.ID)
}
sum := firstNonEmpty(e.Summary, e.Content)
item := FeedItem{
Title: cleanText(e.Title),
Summary: cleanText(sum),
Link: normalizeURL(link),
Source: src,
ImageURL: atomImageURL(e),
}
if item.Title == "" || item.Link == "" {
continue
}
out = append(out, item)
}
if len(out) == 0 {
return nil, fmt.Errorf("empty feed %s", feedURL)
}
return out, nil
}
func rssImageURL(it *rssItem) string {
if it.Enclosure != nil && isImageMIME(it.Enclosure.Type) {
if u := normalizeURL(it.Enclosure.URL); u != "" {
return u
}
}
for i := range it.MediaContent {
m := &it.MediaContent[i]
if isImageMedia(m) {
if u := normalizeURL(m.URL); u != "" {
return u
}
}
}
for i := range it.MediaThumb {
m := &it.MediaThumb[i]
if u := normalizeURL(m.URL); u != "" {
return u
}
}
return firstImgSrc(it.Description)
}
func atomImageURL(e *atomEntry) string {
for i := range e.Links {
l := &e.Links[i]
if strings.EqualFold(l.Rel, "enclosure") || strings.HasPrefix(strings.ToLower(l.Type), "image/") {
if u := normalizeURL(l.Href); u != "" {
return u
}
}
}
return firstImgSrc(firstNonEmpty(e.Summary, e.Content))
}
func isImageMIME(t string) bool {
t = strings.ToLower(strings.TrimSpace(t))
return strings.HasPrefix(t, "image/")
}
func isImageMedia(m *rssMediaContent) bool {
if isImageMIME(m.Type) {
return true
}
return strings.EqualFold(strings.TrimSpace(m.Medium), "image")
}
func firstImgSrc(htmlBody string) string {
m := imgSrcRe.FindStringSubmatch(htmlBody)
if len(m) < 2 {
return ""
}
return normalizeURL(m[1])
}
func atomBestLink(links []atomLink) string {
var alt, self, fallback string
for i := range links {
l := &links[i]
href := strings.TrimSpace(l.Href)
if href == "" {
continue
}
fallback = href
switch strings.ToLower(strings.TrimSpace(l.Rel)) {
case "", "alternate":
if alt == "" {
alt = href
}
case "self":
if self == "" {
self = href
}
}
}
return firstNonEmpty(alt, self, fallback)
}
func FormatPostText(item *FeedItem) string {
title := strings.TrimSpace(item.Title)
sum := truncateRunes(strings.TrimSpace(item.Summary), summaryRunes)
link := strings.TrimSpace(item.Link)
src := strings.TrimSpace(item.Source)
var b strings.Builder
if src != "" {
b.WriteString(src)
b.WriteString("\n")
}
b.WriteString(title)
if sum != "" && !strings.EqualFold(sum, title) {
b.WriteString("\n\n")
b.WriteString(sum)
}
b.WriteString("\n\n")
b.WriteString(link)
text := strings.TrimSpace(b.String())
return truncateRunes(text, 1000)
}
func cleanText(s string) string {
s = html.UnescapeString(s)
s = tagRe.ReplaceAllString(s, " ")
s = strings.Join(strings.Fields(s), " ")
return strings.TrimSpace(s)
}
func truncateRunes(s string, limit int) string {
if limit <= 0 || utf8.RuneCountInString(s) <= limit {
return s
}
runes := []rune(s)
return string(runes[:limit-1]) + "…"
}
func normalizeURL(raw string) string {
raw = strings.TrimSpace(raw)
if raw == "" {
return ""
}
u, err := url.Parse(raw)
if err != nil || u.Scheme == "" || u.Host == "" {
return ""
}
scheme := strings.ToLower(u.Scheme)
if scheme != "http" && scheme != "https" {
return ""
}
u.Fragment = ""
return u.String()
}
func firstNonEmpty(vals ...string) string {
for _, v := range vals {
if strings.TrimSpace(v) != "" {
return strings.TrimSpace(v)
}
}
return ""
}