prompts-core/service/prompt/prompt_files_handle_service.go

package prompt

import (
	"archive/zip"
	"bytes"
	"context"
	"fmt"
	"io"
	"net/http"
	"strings"
	"time"

	"prompts-core/common/util"
	"prompts-core/service/gateway"

	"github.com/gogf/gf/v2/frame/g"
)

// FetchFileTexts 从 URL 列表获取文件内容，支持 zip 内文件
func FetchFileTexts(ctx context.Context, urls []string) map[string]string {
	result := make(map[string]string)

	if len(urls) == 0 {
		return result
	}

	client := &http.Client{
		Timeout: time.Duration(g.Cfg().MustGet(ctx, "userFiles.httpTimeoutSec", 8).Int()) * time.Second,
	}

	for _, rawURL := range urls {
		url := util.SanitizeURL(rawURL)
		if url == "" {
			continue
		}

		if util.IsBannedExtension(url) {
			continue
		}

		if util.IsZipExtension(url) {
			zipTexts := fetchZipFileTexts(ctx, client, url)
			for k, v := range zipTexts {
				result[k] = v
			}
			continue
		}

		text, err := fetchFileContent(ctx, client, url)
		if err != nil {
			continue
		}

		if text == "" {
			continue
		}

		text = util.CleanSymbols(text)
		result[url] = text
	}

	return result
}

// fetchZipFileTexts 下载并解压 zip 文件，提取可读文本内容
func fetchZipFileTexts(ctx context.Context, client *http.Client, url string) map[string]string {
	result := make(map[string]string)

	zipBytes, err := downloadFile(client, url,
		int64(g.Cfg().MustGet(ctx, "userFiles.zipMaxSizeMB", 10).Int())*1024*1024,
	)
	if err != nil {
		return result
	}

	reader, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
	if err != nil {
		return result
	}

	entryMaxSize := int64(g.Cfg().MustGet(ctx, "userFiles.zipEntryMaxSizeKB", 500).Int()) * 1024

	for _, file := range reader.File {
		if file.FileInfo().IsDir() {
			continue
		}

		fileName := file.Name

		if util.IsBannedExtension(fileName) {
			continue
		}

		if util.IsZipExtension(fileName) {
			continue
		}

		rc, err := file.Open()
		if err != nil {
			continue
		}

		content, err := io.ReadAll(io.LimitReader(rc, entryMaxSize))
		rc.Close()
		if err != nil {
			continue
		}

		contentType := http.DetectContentType(content)
		if !util.IsReadableContentType(contentType) {
			continue
		}

		text := util.CleanSymbols(string(content))
		if text == "" {
			continue
		}

		key := url + "::" + fileName
		result[key] = text
	}

	return result
}

// downloadFile 下载文件，限制最大大小
func downloadFile(client *http.Client, url string, maxSize int64) ([]byte, error) {
	req, err := http.NewRequest(http.MethodGet, url, nil)
	if err != nil {
		return nil, err
	}

	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
	}

	return io.ReadAll(io.LimitReader(resp.Body, maxSize))
}

// fetchFileContent 获取单个文本文件内容
func fetchFileContent(ctx context.Context, client *http.Client, url string) (string, error) {
	req, err := http.NewRequest(http.MethodGet, url, nil)
	if err != nil {
		return "", err
	}

	resp, err := client.Do(req)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		return "", fmt.Errorf("HTTP %d", resp.StatusCode)
	}

	contentType := resp.Header.Get("Content-Type")
	if !util.IsReadableContentType(contentType) {
		return "", fmt.Errorf("unreadable content-type: %s", contentType)
	}

	body, err := io.ReadAll(
		io.LimitReader(resp.Body,
			int64(g.Cfg().MustGet(ctx, "userFiles.textFileMaxSizeKB", 500).Int())*1024,
		),
	)
	if err != nil {
		return "", err
	}

	return strings.TrimSpace(string(body)), nil
}

// SkillMdContent 根据 skillName 获取 zip 内所有 md 文件拼接内容
func SkillMdContent(ctx context.Context, skillName string) string {
	skillResp, err := gateway.GetSkillUser(ctx, skillName)
	if err != nil {
		return ""
	}

	fullUrl := skillResp.ImgAddressPrefix + skillResp.FileUrl

	client := &http.Client{
		Timeout: time.Duration(g.Cfg().MustGet(ctx, "skillFiles.httpTimeoutSec", 30).Int()) * time.Second,
	}

	zipBytes, err := downloadFile(client, fullUrl,
		int64(g.Cfg().MustGet(ctx, "skillFiles.zipMaxSizeMB", 10).Int())*1024*1024,
	)
	if err != nil {
		return ""
	}

	mdContents, err := extractMdFiles(ctx, zipBytes)
	if err != nil {
		return ""
	}

	if len(mdContents) == 0 {
		return ""
	}

	var builder strings.Builder
	builder.WriteString(fmt.Sprintf("# Skill: %s\n\n", skillResp.Name))
	if skillResp.Description != "" {
		builder.WriteString(fmt.Sprintf("> %s\n\n", skillResp.Description))
	}

	for fileName, content := range mdContents {
		builder.WriteString(fmt.Sprintf("## %s\n\n", fileName))
		builder.WriteString(content)
		builder.WriteString("\n\n---\n\n")
	}

	return strings.TrimSpace(builder.String())
}

// extractMdFiles 解压 zip 并提取所有 .md 文件内容
func extractMdFiles(ctx context.Context, zipBytes []byte) (map[string]string, error) {
	result := make(map[string]string)

	reader, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
	if err != nil {
		return nil, err
	}

	entryMaxSize := int64(g.Cfg().MustGet(ctx, "skillFiles.mdMaxSizeKB", 500).Int()) * 1024

	for _, file := range reader.File {
		if file.FileInfo().IsDir() {
			continue
		}

		if !strings.HasSuffix(strings.ToLower(file.Name), ".md") {
			continue
		}

		rc, err := file.Open()
		if err != nil {
			continue
		}

		content, err := io.ReadAll(io.LimitReader(rc, entryMaxSize))
		rc.Close()
		if err != nil {
			continue
		}

		if len(content) > 0 {
			result[file.Name] = strings.TrimSpace(string(content))
		}
	}

	return result, nil
}