feat(llm): support llm-bench in llm (#25103)

cmd/climc/shell/llm/llm_benchmark.go

@@ -0,0 +1,55 @@
+package llm
+
+import (
+	"io"
+	"os"
+
+	"github.com/cheggaaa/pb/v3"
+
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/cmd/climc/shell"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	modules "yunion.io/x/onecloud/pkg/mcclient/modules/llm"
+	options "yunion.io/x/onecloud/pkg/mcclient/options/llm"
+)
+
+func init() {
+	cmd := shell.NewResourceCmd(&modules.LLMBenchmarks)
+	cmd.List(new(options.LLMBenchmarkListOptions))
+	cmd.Show(new(options.LLMBenchmarkShowOptions))
+	cmd.Create(new(options.LLMBenchmarkCreateOptions))
+	cmd.Update(new(options.LLMBenchmarkUpdateOptions))
+	cmd.Delete(new(options.LLMBenchmarkDeleteOptions))
+	cmd.Perform("copy", new(options.LLMBenchmarkCopyOptions))
+	cmd.Perform("retest", new(options.LLMBenchmarkRetestOptions))
+	cmd.Perform("stop", new(options.LLMBenchmarkStopOptions))
+
+	shell.R(new(options.LLMBenchmarkArtifactOptions), "llm-benchmark-artifact", "Download benchmark artifact", func(s *mcclient.ClientSession, args *options.LLMBenchmarkArtifactOptions) error {
+		src, size, err := modules.LLMBenchmarks.Artifact(s, args.ID, args.Type)
+		if err != nil {
+			return errors.Wrap(err, "download artifact")
+		}
+		defer src.Close()
+		return writeBenchmarkArtifact(src, size, args.Output)
+	})
+}
+
+func writeBenchmarkArtifact(src io.Reader, size int64, output string) error {
+	if output == "" {
+		_, err := io.Copy(os.Stdout, src)
+		return err
+	}
+	f, err := os.Create(output)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	if size < 0 {
+		_, err = io.Copy(f, src)
+		return err
+	}
+	bar := pb.Full.Start64(size)
+	_, err = io.Copy(f, bar.NewProxyReader(src))
+	return err
+}

cmd/climc/shell/llm/llm_benchmark_package.go

@@ -0,0 +1,16 @@
+package llm
+
+import (
+	"yunion.io/x/onecloud/cmd/climc/shell"
+	modules "yunion.io/x/onecloud/pkg/mcclient/modules/llm"
+	options "yunion.io/x/onecloud/pkg/mcclient/options/llm"
+)
+
+func init() {
+	cmd := shell.NewResourceCmd(&modules.LLMBenchmarkPackages)
+	cmd.List(new(options.LLMBenchmarkPackageListOptions))
+	cmd.Show(new(options.LLMBenchmarkPackageShowOptions))
+	cmd.Create(new(options.LLMBenchmarkPackageCreateOptions))
+	cmd.Delete(new(options.LLMBenchmarkPackageDeleteOptions))
+	cmd.PerformClass("import", new(options.LLMBenchmarkPackageImportOptions))
+}

pkg/apigateway/handler/llm_benchmark_artifact.go

@@ -0,0 +1,75 @@
+package handler
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/appctx"
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	llmmodules "yunion.io/x/onecloud/pkg/mcclient/modules/llm"
+)
+
+func llmBenchmarkArtifactsHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	proxyLLMBenchmarkArtifact(ctx, w, r, "")
+}
+
+func llmBenchmarkArtifactHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	kind := appctx.AppContextParams(ctx)["<type>"]
+	if kind == "" {
+		httperrors.MissingParameterError(ctx, w, "type")
+		return
+	}
+	proxyLLMBenchmarkArtifact(ctx, w, r, kind)
+}
+
+func proxyLLMBenchmarkArtifact(ctx context.Context, w http.ResponseWriter, r *http.Request, kind string) {
+	id := appctx.AppContextParams(ctx)["<id>"]
+	path := llmBenchmarkArtifactsBackendPath(id)
+	if kind != "" {
+		path = llmBenchmarkArtifactBackendPath(id, kind)
+	}
+
+	s := auth.GetSession(ctx, AppContextToken(ctx), FetchRegion(r))
+	resp, err := s.RawVersionRequest(
+		llmmodules.LLMBenchmarks.ServiceType(),
+		llmmodules.LLMBenchmarks.EndpointType(),
+		"GET",
+		path,
+		nil,
+		nil,
+	)
+	if err != nil {
+		httperrors.GeneralServerError(ctx, w, errors.Wrap(err, "request backend"))
+		return
+	}
+	defer resp.Body.Close()
+
+	copyHTTPHeader(w.Header(), resp.Header)
+	w.WriteHeader(resp.StatusCode)
+	if _, err := io.Copy(w, resp.Body); err != nil {
+		log.Errorf("copy benchmark artifact response: %v", err)
+	}
+}
+
+func llmBenchmarkArtifactsBackendPath(id string) string {
+	return fmt.Sprintf("/llm_benchmarks/%s/artifacts", url.PathEscape(id))
+}
+
+func llmBenchmarkArtifactBackendPath(id string, kind string) string {
+	return fmt.Sprintf("%s/%s", llmBenchmarkArtifactsBackendPath(id), url.PathEscape(kind))
+}
+
+func copyHTTPHeader(dst http.Header, src http.Header) {
+	for k, vv := range src {
+		for _, v := range vv {
+			dst.Add(k, v)
+		}
+	}
+}

pkg/apigateway/handler/misc.go

@@ -122,6 +122,10 @@ func (h *MiscHandler) Bind(app *appsrv.Application) {
 	app.AddHandler(GET, prefix+"mcp-servers-config", mcpServersConfigHandler)
 	// mcp agent default MCP server tools (options.MCPServerURL only, no mcp_agent entry)
 	app.AddHandler(GET, prefix+"default-mcp-tools", FetchAuthToken(mcpAgentDefaultToolsHandler))
+
+	benchArtifactPrefix := prefix + "llm_benchmarks/<id>/artifacts"
+	app.AddHandler(GET, benchArtifactPrefix, FetchAuthToken(llmBenchmarkArtifactsHandler))
+	app.AddHandler(GET, benchArtifactPrefix+"/<type>", FetchAuthToken(llmBenchmarkArtifactHandler))
 }
 
 func UploadHandlerInfo(method, prefix string, handler func(context.Context, http.ResponseWriter, *http.Request)) *appsrv.SHandlerInfo {

pkg/apis/llm/image.go

@@ -18,6 +18,7 @@ const (
 	LLM_IMAGE_TYPE_HERMES_AGENT LLMImageType = "hermes-agent"
 	LLM_IMAGE_TYPE_LLM_ROUTER   LLMImageType = "llm-router"
 	LLM_IMAGE_TYPE_DESKTOP      LLMImageType = "desktop"
+	LLM_IMAGE_TYPE_BENCHMARK    LLMImageType = "benchmark"
 )
 
 var (
@@ -31,6 +32,7 @@ var (
 		string(LLM_IMAGE_TYPE_HERMES_AGENT),
 		string(LLM_IMAGE_TYPE_LLM_ROUTER),
 		string(LLM_IMAGE_TYPE_DESKTOP),
+		string(LLM_IMAGE_TYPE_BENCHMARK),
 	)
 )
 

pkg/apis/llm/llm_benchmark.go

@@ -0,0 +1,166 @@
+package llm
+
+import (
+	"reflect"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/gotypes"
+
+	"yunion.io/x/onecloud/pkg/apis"
+)
+
+const (
+	LLMBenchmarkStatePending    = "pending"
+	LLMBenchmarkStateQueued     = "queued"
+	LLMBenchmarkStateValidating = "validating"
+	LLMBenchmarkStateRunning    = "running"
+	LLMBenchmarkStateCompleted  = "completed"
+	LLMBenchmarkStateStopped    = "stopped"
+	LLMBenchmarkStateError      = "error"
+)
+
+const (
+	LLMBenchmarkDatasetSyntheticText = "synthetic_text"
+	LLMBenchmarkDatasetPackage       = "benchmark_package"
+	LLMBenchmarkProfileConstant      = "constant"
+)
+
+const (
+	LLMBenchmarkEvaluationStateEvaluating = "evaluating"
+	LLMBenchmarkEvaluationStateCompleted  = "completed"
+	LLMBenchmarkEvaluationStateSkipped    = "skipped"
+	LLMBenchmarkEvaluationStateError      = "error"
+)
+
+const (
+	LLMBenchmarkArtifactStorageLocal = "local"
+	LLMBenchmarkArtifactStorageMinio = "minio"
+)
+
+const (
+	LLMBenchmarkDefaultRequestFormat = "/v1/chat/completions"
+	LLMBenchmarkDefaultImage         = "registry.cn-beijing.aliyuncs.com/cloudpods/guidellm:v0.7.0-amd64"
+)
+
+type LLMBenchmarkCreateInput struct {
+	apis.VirtualResourceCreateInput
+
+	LLMId            string `json:"llm_id"`
+	BenchmarkImage   string `json:"benchmark_image,omitempty"`
+	BenchmarkPackage string `json:"benchmark_package,omitempty"`
+
+	RequestFormat string `json:"request_format,omitempty"`
+	Model         string `json:"model,omitempty"`
+
+	Profile     string `json:"profile,omitempty"`
+	RequestRate int    `json:"request_rate,omitempty"`
+
+	TotalRequests      int `json:"total_requests,omitempty"`
+	MaxDurationSeconds int `json:"max_duration_seconds,omitempty"`
+	MaxErrors          int `json:"max_errors,omitempty"`
+
+	DatasetName         string `json:"dataset_name,omitempty"`
+	DatasetInputTokens  int    `json:"dataset_input_tokens,omitempty"`
+	DatasetOutputTokens int    `json:"dataset_output_tokens,omitempty"`
+	DatasetPath         string `json:"dataset_path,omitempty"`
+
+	LLMDeploymentId    string `json:"llm_deployment_id,omitempty"`
+	LLMSkuId           string `json:"llm_sku_id,omitempty"`
+	LLMImageId         string `json:"llm_image_id,omitempty"`
+	BenchmarkPackageId string `json:"benchmark_package_id,omitempty"`
+	Backend            string `json:"backend,omitempty"`
+	TargetUrl          string `json:"target_url,omitempty"`
+	WorkDir            string `json:"work_dir,omitempty"`
+	TargetSnapshot     string `json:"target_snapshot,omitempty"`
+	GuideLLMSpec       string `json:"guide_llm_spec,omitempty"`
+}
+
+type LLMBenchmarkCopyInput struct {
+	Name            string `json:"name"`
+	LLMDeploymentId string `json:"llm_deployment_id"`
+
+	Description         *string `json:"description,omitempty"`
+	Model               *string `json:"model,omitempty"`
+	RequestRate         *int    `json:"request_rate,omitempty"`
+	TotalRequests       *int    `json:"total_requests,omitempty"`
+	MaxDurationSeconds  *int    `json:"max_duration_seconds,omitempty"`
+	MaxErrors           *int    `json:"max_errors,omitempty"`
+	DatasetInputTokens  *int    `json:"dataset_input_tokens,omitempty"`
+	DatasetOutputTokens *int    `json:"dataset_output_tokens,omitempty"`
+}
+
+type LLMBenchmarkUpdateInput struct {
+	apis.VirtualResourceBaseUpdateInput
+
+	Model               *string `json:"model,omitempty"`
+	RequestRate         *int    `json:"request_rate,omitempty"`
+	TotalRequests       *int    `json:"total_requests,omitempty"`
+	MaxDurationSeconds  *int    `json:"max_duration_seconds,omitempty"`
+	MaxErrors           *int    `json:"max_errors,omitempty"`
+	DatasetInputTokens  *int    `json:"dataset_input_tokens,omitempty"`
+	DatasetOutputTokens *int    `json:"dataset_output_tokens,omitempty"`
+}
+
+type LLMBenchmarkRetestInput struct{}
+
+type LLMBenchmarkListInput struct {
+	apis.VirtualResourceListInput
+
+	LLMId           string `json:"llm_id"`
+	LLMDeploymentId string `json:"llm_deployment_id"`
+	State           string `json:"state"`
+}
+
+type LLMBenchmarkDatasetPreflight struct {
+	State              string  `json:"state"`
+	ExpectedSamples    int     `json:"expected_samples"`
+	ActualSamples      int     `json:"actual_samples"`
+	Successful         int     `json:"successful"`
+	Errored            int     `json:"errored"`
+	ErrorRate          float64 `json:"error_rate"`
+	LatencyMeanSeconds float64 `json:"latency_mean_sec"`
+	Message            string  `json:"message"`
+}
+
+func (p *LLMBenchmarkDatasetPreflight) String() string {
+	return jsonutils.Marshal(p).String()
+}
+
+func (p *LLMBenchmarkDatasetPreflight) IsZero() bool {
+	return p == nil || *p == (LLMBenchmarkDatasetPreflight{})
+}
+
+type LLMBenchmarkDatasetEvaluation struct {
+	State        string  `json:"state"`
+	AnswerColumn string  `json:"answer_column,omitempty"`
+	RequestTotal int     `json:"request_total"`
+	Evaluated    int     `json:"evaluated"`
+	Correct      int     `json:"correct"`
+	Incorrect    int     `json:"incorrect"`
+	Unscored     int     `json:"unscored"`
+	Accuracy     float64 `json:"accuracy"`
+	Message      string  `json:"message,omitempty"`
+}
+
+func (e *LLMBenchmarkDatasetEvaluation) String() string {
+	return jsonutils.Marshal(e).String()
+}
+
+func (e *LLMBenchmarkDatasetEvaluation) IsZero() bool {
+	return e == nil || *e == (LLMBenchmarkDatasetEvaluation{})
+}
+
+func init() {
+	gotypes.RegisterSerializable(reflect.TypeOf(new(LLMBenchmarkDatasetPreflight)), func() gotypes.ISerializable {
+		return new(LLMBenchmarkDatasetPreflight)
+	})
+	gotypes.RegisterSerializable(reflect.TypeOf(new(LLMBenchmarkDatasetEvaluation)), func() gotypes.ISerializable {
+		return new(LLMBenchmarkDatasetEvaluation)
+	})
+}
+
+type LLMBenchmarkDetails struct {
+	apis.VirtualResourceDetails
+
+	LLMDeployment string `json:"llm_deployment"`
+}

pkg/apis/llm/llm_benchmark_package.go

@@ -0,0 +1,42 @@
+package llm
+
+import "yunion.io/x/onecloud/pkg/apis"
+
+const (
+	LLMBenchmarkPackageSourceHuggingFace   = "huggingface"
+	LLMBenchmarkPackageFormatGuideLLMJSONL = "guidellm_jsonl"
+
+	LLMBenchmarkPackageMountBase = "/data/benchmark-packages"
+)
+
+type LLMBenchmarkPackageCreateInput struct {
+	apis.SharableVirtualResourceCreateInput
+
+	Source       string `json:"source,omitempty"`
+	RepoId       string `json:"repo_id,omitempty"`
+	Revision     string `json:"revision,omitempty"`
+	FilePath     string `json:"file_path,omitempty"`
+	Format       string `json:"format,omitempty"`
+	AnswerColumn string `json:"answer_column,omitempty"`
+
+	ImageId      string `json:"image_id,omitempty"`
+	Size         int64  `json:"size,omitempty"`
+	ActualSizeMb int32  `json:"actual_size_mb,omitempty"`
+
+	MountPath   string `json:"mount_path,omitempty"`
+	DatasetPath string `json:"dataset_path,omitempty"`
+	Manifest    string `json:"manifest,omitempty"`
+}
+
+type LLMBenchmarkPackageImportInput struct {
+	LLMBenchmarkPackageCreateInput
+	BenchmarkSpec *LLMBenchmarkCreateInput `json:"benchmark_spec,omitempty"`
+}
+
+type LLMBenchmarkPackageListInput struct {
+	apis.SharableVirtualResourceListInput
+
+	Source string `json:"source"`
+	RepoId string `json:"repo_id"`
+	Format string `json:"format"`
+}

pkg/llm/benchmark/artifact_store.go

@@ -0,0 +1,286 @@
+package benchmark
+
+import (
+	"context"
+	"errors"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	smithyhttp "github.com/aws/smithy-go/transport/http"
+
+	api "yunion.io/x/onecloud/pkg/apis/llm"
+)
+
+type ArtifactStoreOptions struct {
+	Endpoint  string
+	AccessKey string
+	SecretKey string
+	Bucket    string
+	Secure    bool
+	Prefix    string
+}
+
+type ArtifactStore struct {
+	opts    ArtifactStoreOptions
+	client  *s3.Client
+	initErr error
+}
+
+var defaultArtifactStore = NewArtifactStore(ArtifactStoreOptions{})
+
+func ConfigureArtifactStore(opts ArtifactStoreOptions) {
+	defaultArtifactStore = NewArtifactStore(opts)
+}
+
+func DefaultArtifactStore() *ArtifactStore {
+	return defaultArtifactStore
+}
+
+func NewArtifactStore(opts ArtifactStoreOptions) *ArtifactStore {
+	opts.Endpoint = strings.TrimSpace(opts.Endpoint)
+	opts.Prefix = strings.Trim(strings.TrimSpace(opts.Prefix), "/")
+	ret := &ArtifactStore{opts: opts}
+	if opts.Endpoint == "" {
+		return ret
+	}
+	if opts.AccessKey == "" || opts.SecretKey == "" || opts.Bucket == "" {
+		ret.initErr = errors.New("missing MinIO/S3 artifact config")
+		return ret
+	}
+	endpoint := opts.Endpoint
+	if !strings.HasPrefix(endpoint, "http://") && !strings.HasPrefix(endpoint, "https://") {
+		if opts.Secure {
+			endpoint = "https://" + endpoint
+		} else {
+			endpoint = "http://" + endpoint
+		}
+	}
+	ret.client = s3.NewFromConfig(aws.Config{
+		Region:       "us-east-1",
+		Credentials:  credentials.NewStaticCredentialsProvider(opts.AccessKey, opts.SecretKey, ""),
+		BaseEndpoint: aws.String(endpoint),
+	}, func(o *s3.Options) {
+		o.UsePathStyle = true
+	})
+	return ret
+}
+
+func (s *ArtifactStore) Enabled() bool {
+	return s != nil && s.opts.Endpoint != ""
+}
+
+func (s *ArtifactStore) benchmarkPrefix(projectID, benchmarkID string) string {
+	return path.Join(s.opts.Prefix, projectID, benchmarkID) + "/"
+}
+
+func (s *ArtifactStore) objectKey(projectID, benchmarkID, local string) string {
+	return path.Join(s.opts.Prefix, projectID, benchmarkID, filepath.Base(local))
+}
+
+func artifactURI(bucket, key string) string {
+	return "s3://" + bucket + "/" + key
+}
+
+func parseArtifactURI(location string) (string, string, error) {
+	parsed, err := url.Parse(location)
+	if err != nil || parsed.Scheme != "s3" || parsed.Host == "" {
+		return "", "", errors.New("invalid artifact S3 URI")
+	}
+	key := strings.TrimPrefix(parsed.EscapedPath(), "/")
+	key, err = url.PathUnescape(key)
+	if err != nil || key == "" {
+		return "", "", errors.New("invalid artifact S3 object key")
+	}
+	return parsed.Host, key, nil
+}
+
+func cloneArtifactPaths(paths map[string]string) map[string]string {
+	ret := make(map[string]string, len(paths))
+	for kind, location := range paths {
+		if location != "" {
+			ret[kind] = location
+		}
+	}
+	return ret
+}
+
+func artifactNotFound(err error) bool {
+	var responseError *smithyhttp.ResponseError
+	return errors.As(err, &responseError) && responseError.HTTPStatusCode() == http.StatusNotFound
+}
+
+func (s *ArtifactStore) Persist(
+	ctx context.Context,
+	projectID, benchmarkID string,
+	files map[string]string,
+) (map[string]string, string, error) {
+	local := cloneArtifactPaths(files)
+	if len(local) == 0 {
+		return local, "", nil
+	}
+	if !s.Enabled() {
+		return local, api.LLMBenchmarkArtifactStorageLocal, nil
+	}
+	if s.initErr != nil {
+		return local, api.LLMBenchmarkArtifactStorageLocal, s.initErr
+	}
+	if err := ensureArtifactBucket(ctx, s.client, s.opts.Bucket); err != nil {
+		return local, api.LLMBenchmarkArtifactStorageLocal, err
+	}
+
+	kinds := make([]string, 0, len(local))
+	for kind := range local {
+		kinds = append(kinds, kind)
+	}
+	sort.Strings(kinds)
+	remote := make(map[string]string, len(local))
+	uploaded := make([]string, 0, len(local))
+	for _, kind := range kinds {
+		file, err := os.Open(local[kind])
+		if err == nil {
+			key := s.objectKey(projectID, benchmarkID, local[kind])
+			uploaded = append(uploaded, key)
+			_, err = s.client.PutObject(ctx, &s3.PutObjectInput{
+				Bucket: aws.String(s.opts.Bucket),
+				Key:    aws.String(key),
+				Body:   file,
+			})
+			closeErr := file.Close()
+			if err == nil {
+				err = closeErr
+			}
+			if err == nil {
+				remote[kind] = artifactURI(s.opts.Bucket, key)
+			}
+		}
+		if err != nil {
+			for _, key := range uploaded {
+				_, _ = s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
+					Bucket: aws.String(s.opts.Bucket),
+					Key:    aws.String(key),
+				})
+			}
+			return local, api.LLMBenchmarkArtifactStorageLocal, err
+		}
+	}
+	return remote, api.LLMBenchmarkArtifactStorageMinio, nil
+}
+
+func ensureArtifactBucket(ctx context.Context, client *s3.Client, bucket string) error {
+	if _, err := client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: aws.String(bucket)}); err == nil {
+		return nil
+	}
+	_, err := client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(bucket)})
+	return err
+}
+
+func (s *ArtifactStore) RemoveLocal(files map[string]string) error {
+	var firstErr error
+	for _, file := range files {
+		if file == "" {
+			continue
+		}
+		if err := os.Remove(file); err != nil && !os.IsNotExist(err) && firstErr == nil {
+			firstErr = err
+		}
+	}
+	return firstErr
+}
+
+func (s *ArtifactStore) Exists(ctx context.Context, location string) (bool, error) {
+	if location == "" {
+		return false, nil
+	}
+	if !strings.HasPrefix(location, "s3://") {
+		_, err := os.Stat(location)
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return err == nil, err
+	}
+	if s == nil || s.client == nil {
+		if s != nil && s.initErr != nil {
+			return false, s.initErr
+		}
+		return false, errors.New("MinIO/S3 artifact store is disabled")
+	}
+	bucket, key, err := parseArtifactURI(location)
+	if err != nil {
+		return false, err
+	}
+	_, err = s.client.HeadObject(ctx, &s3.HeadObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
+	})
+	if artifactNotFound(err) {
+		return false, nil
+	}
+	return err == nil, err
+}
+
+func (s *ArtifactStore) Open(ctx context.Context, location string) (io.ReadCloser, error) {
+	if !strings.HasPrefix(location, "s3://") {
+		return os.Open(location)
+	}
+	if s == nil || s.client == nil {
+		if s != nil && s.initErr != nil {
+			return nil, s.initErr
+		}
+		return nil, errors.New("MinIO/S3 artifact store is disabled")
+	}
+	bucket, key, err := parseArtifactURI(location)
+	if err != nil {
+		return nil, err
+	}
+	out, err := s.client.GetObject(ctx, &s3.GetObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
+	})
+	if err != nil {
+		return nil, err
+	}
+	return out.Body, nil
+}
+
+func (s *ArtifactStore) DeleteBenchmark(ctx context.Context, projectID, benchmarkID string) error {
+	if !s.Enabled() {
+		return nil
+	}
+	if s.initErr != nil {
+		return s.initErr
+	}
+	input := &s3.ListObjectsV2Input{
+		Bucket: aws.String(s.opts.Bucket),
+		Prefix: aws.String(s.benchmarkPrefix(projectID, benchmarkID)),
+	}
+	for {
+		out, err := s.client.ListObjectsV2(ctx, input)
+		if artifactNotFound(err) {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+		for _, object := range out.Contents {
+			if _, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
+				Bucket: aws.String(s.opts.Bucket),
+				Key:    object.Key,
+			}); err != nil {
+				return err
+			}
+		}
+		if !aws.ToBool(out.IsTruncated) || out.NextContinuationToken == nil {
+			return nil
+		}
+		input.ContinuationToken = out.NextContinuationToken
+	}
+}

pkg/llm/benchmark/doc.go

@@ -0,0 +1 @@
+package benchmark // import "yunion.io/x/onecloud/pkg/llm/benchmark"

pkg/llm/benchmark/evaluation.go

@@ -0,0 +1,772 @@
+package benchmark
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/csv"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"math/big"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+
+	api "yunion.io/x/onecloud/pkg/apis/llm"
+)
+
+const evaluationExcerptRunes = 1024
+
+var answerColumnCandidates = []string{
+	"reference_answer",
+	"answer",
+	"solution",
+	"target",
+	"label",
+}
+
+var promptColumnCandidates = []string{
+	"prompt",
+	"instruction",
+	"question",
+	"input",
+	"context",
+	"content",
+	"conversation",
+	"turn",
+	"text",
+}
+
+var decimalNumberPattern = regexp.MustCompile(`^[+-]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][+-]?[0-9]+)?$`)
+
+type EvaluationInput struct {
+	DatasetPath   string
+	BenchmarkPath string
+	AnswerColumn  string
+	OutputDir     string
+}
+
+type EvaluationResult struct {
+	Summary    api.LLMBenchmarkDatasetEvaluation
+	ResultJSON string
+	ResultCSV  string
+	LogPath    string
+}
+
+type evaluationExcerpt struct {
+	Text      string `json:"text"`
+	RuneCount int    `json:"rune_count"`
+	SHA256    string `json:"sha256"`
+	Truncated bool   `json:"truncated"`
+}
+
+type answerValue struct {
+	scorable bool
+	key      string
+	textHash string
+	number   string
+	excerpt  evaluationExcerpt
+}
+
+type datasetEntry struct {
+	index    int
+	answer   answerValue
+	conflict bool
+}
+
+const (
+	evaluationItemCorrect   = "correct"
+	evaluationItemIncorrect = "incorrect"
+	evaluationItemUnscored  = "unscored"
+)
+
+type evaluationItem struct {
+	RequestID        string             `json:"request_id"`
+	DatasetIndex     *int               `json:"dataset_index,omitempty"`
+	State            string             `json:"state"`
+	MatchType        string             `json:"match_type,omitempty"`
+	ReferenceExcerpt *evaluationExcerpt `json:"reference_excerpt,omitempty"`
+	PromptExcerpt    *evaluationExcerpt `json:"prompt_excerpt,omitempty"`
+	OutputExcerpt    *evaluationExcerpt `json:"output_excerpt,omitempty"`
+	Message          string             `json:"message,omitempty"`
+}
+
+var guideRequestArrayStates = map[string]string{
+	"successful_requests": "successful",
+	"successful":          "successful",
+	"errored_requests":    "errored",
+	"errored":             "errored",
+	"incomplete_requests": "incomplete",
+	"incomplete":          "incomplete",
+	"timed_out_requests":  "timed_out",
+	"timeout_requests":    "timed_out",
+}
+
+func normalizeWhitespace(value string) string {
+	value = strings.TrimSpace(value)
+	var out strings.Builder
+	pendingSpace := false
+	for _, r := range value {
+		if unicode.IsSpace(r) {
+			pendingSpace = true
+			continue
+		}
+		if pendingSpace && out.Len() > 0 {
+			out.WriteByte(' ')
+		}
+		pendingSpace = false
+		out.WriteRune(r)
+	}
+	return out.String()
+}
+
+func canonicalFold(value string) string {
+	var out strings.Builder
+	for _, r := range value {
+		min := r
+		for next := unicode.SimpleFold(r); next != r; next = unicode.SimpleFold(next) {
+			if next < min {
+				min = next
+			}
+		}
+		out.WriteRune(min)
+	}
+	return out.String()
+}
+
+func sha256String(value string) string {
+	sum := sha256.Sum256([]byte(value))
+	return hex.EncodeToString(sum[:])
+}
+
+func parseExactNumber(value string) (string, bool) {
+	if !decimalNumberPattern.MatchString(value) {
+		return "", false
+	}
+	number, ok := new(big.Rat).SetString(value)
+	if !ok {
+		return "", false
+	}
+	return number.RatString(), true
+}
+
+func makeEvaluationExcerpt(value string) evaluationExcerpt {
+	runeCount := utf8.RuneCountInString(value)
+	text := value
+	truncated := runeCount > evaluationExcerptRunes
+	if truncated {
+		text = string([]rune(value)[:evaluationExcerptRunes])
+	}
+	return evaluationExcerpt{
+		Text:      text,
+		RuneCount: runeCount,
+		SHA256:    sha256String(value),
+		Truncated: truncated,
+	}
+}
+
+func scalarText(raw json.RawMessage) (string, bool) {
+	if len(raw) == 0 {
+		return "", false
+	}
+	decoder := json.NewDecoder(bytes.NewReader(raw))
+	decoder.UseNumber()
+	var value interface{}
+	if err := decoder.Decode(&value); err != nil {
+		return "", false
+	}
+	switch value := value.(type) {
+	case string:
+		return value, true
+	case json.Number:
+		return value.String(), true
+	case bool:
+		return strconv.FormatBool(value), true
+	default:
+		return "", false
+	}
+}
+
+func newAnswerValue(raw json.RawMessage) answerValue {
+	text, ok := scalarText(raw)
+	if !ok {
+		return answerValue{key: "unscored"}
+	}
+	normalized := normalizeWhitespace(text)
+	ret := answerValue{
+		scorable: true,
+		textHash: sha256String(canonicalFold(normalized)),
+		excerpt:  makeEvaluationExcerpt(text),
+	}
+	if number, ok := parseExactNumber(normalized); ok {
+		ret.number = number
+		ret.key = "number:" + number
+	} else {
+		ret.key = "text:" + ret.textHash
+	}
+	return ret
+}
+
+func matchAnswer(reference answerValue, output string) (bool, string) {
+	normalized := normalizeWhitespace(output)
+	if reference.number != "" {
+		if number, ok := parseExactNumber(normalized); ok {
+			return number == reference.number, "number"
+		}
+	}
+	return sha256String(canonicalFold(normalized)) == reference.textHash, "text"
+}
+
+func forEachDatasetRow(
+	ctx context.Context,
+	path string,
+	visit func(index int, row map[string]json.RawMessage) error,
+) error {
+	file, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+	decoder := json.NewDecoder(file)
+	for index := 0; ; index++ {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+		}
+		row := map[string]json.RawMessage{}
+		if err := decoder.Decode(&row); err != nil {
+			if errors.Is(err, io.EOF) {
+				return nil
+			}
+			return fmt.Errorf("decode dataset row %d: %w", index, err)
+		}
+		if err := visit(index, row); err != nil {
+			return err
+		}
+	}
+}
+
+func detectAnswerColumn(ctx context.Context, path, explicit string) (string, string, error) {
+	explicit = strings.TrimSpace(explicit)
+	if explicit != "" {
+		found := false
+		err := forEachDatasetRow(ctx, path, func(_ int, row map[string]json.RawMessage) error {
+			if _, ok := row[explicit]; ok {
+				found = true
+			}
+			return nil
+		})
+		if err != nil {
+			return "", "", err
+		}
+		if !found {
+			return "", fmt.Sprintf("answer column %q not found", explicit), nil
+		}
+		return explicit, "", nil
+	}
+
+	found := map[string]bool{}
+	err := forEachDatasetRow(ctx, path, func(_ int, row map[string]json.RawMessage) error {
+		for _, candidate := range answerColumnCandidates {
+			if _, ok := row[candidate]; ok {
+				found[candidate] = true
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return "", "", err
+	}
+	matches := make([]string, 0, len(found))
+	for _, candidate := range answerColumnCandidates {
+		if found[candidate] {
+			matches = append(matches, candidate)
+		}
+	}
+	switch len(matches) {
+	case 0:
+		return "", "no answer column found", nil
+	case 1:
+		return matches[0], "", nil
+	default:
+		return "", "multiple answer columns found: " + strings.Join(matches, ", "), nil
+	}
+}
+
+func firstStringField(row map[string]json.RawMessage, candidates []string) (string, bool) {
+	for _, candidate := range candidates {
+		raw, ok := row[candidate]
+		if !ok {
+			continue
+		}
+		var value string
+		if json.Unmarshal(raw, &value) == nil {
+			return value, true
+		}
+	}
+	return "", false
+}
+
+func promptDigest(prompt string) string {
+	return sha256String(normalizeWhitespace(prompt))
+}
+
+func buildDatasetIndex(ctx context.Context, path, answerColumn string) (map[string]*datasetEntry, error) {
+	// ponytail: the index is bounded by total_requests; move it to disk only if the 100k-request ceiling is proven too large.
+	index := map[string]*datasetEntry{}
+	err := forEachDatasetRow(ctx, path, func(rowIndex int, row map[string]json.RawMessage) error {
+		prompt, ok := firstStringField(row, promptColumnCandidates)
+		if !ok {
+			return nil
+		}
+		answer := newAnswerValue(row[answerColumn])
+		key := promptDigest(prompt)
+		if existing := index[key]; existing != nil {
+			if existing.answer.key != answer.key {
+				existing.conflict = true
+			}
+			return nil
+		}
+		index[key] = &datasetEntry{
+			index:  rowIndex,
+			answer: answer,
+		}
+		return nil
+	})
+	return index, err
+}
+
+func walkGuideLLMRequests(
+	ctx context.Context,
+	path string,
+	visit func(state string, raw json.RawMessage) error,
+) error {
+	file, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+	decoder := json.NewDecoder(file)
+	return walkGuideLLMValue(ctx, decoder, "", visit)
+}
+
+func walkGuideLLMValue(
+	ctx context.Context,
+	decoder *json.Decoder,
+	arrayState string,
+	visit func(state string, raw json.RawMessage) error,
+) error {
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	default:
+	}
+	token, err := decoder.Token()
+	if err != nil {
+		return err
+	}
+	delimiter, ok := token.(json.Delim)
+	if !ok {
+		return nil
+	}
+	if arrayState != "" && delimiter != '[' {
+		arrayState = ""
+	}
+	switch delimiter {
+	case '{':
+		for decoder.More() {
+			keyToken, err := decoder.Token()
+			if err != nil {
+				return err
+			}
+			key, ok := keyToken.(string)
+			if !ok {
+				return errors.New("GuideLLM object key is not a string")
+			}
+			if err := walkGuideLLMValue(ctx, decoder, guideRequestArrayStates[key], visit); err != nil {
+				return err
+			}
+		}
+		_, err = decoder.Token()
+		return err
+	case '[':
+		if arrayState != "" {
+			for decoder.More() {
+				var raw json.RawMessage
+				if err := decoder.Decode(&raw); err != nil {
+					return err
+				}
+				if err := visit(arrayState, raw); err != nil {
+					return err
+				}
+			}
+			_, err = decoder.Token()
+			return err
+		}
+		for decoder.More() {
+			if err := walkGuideLLMValue(ctx, decoder, "", visit); err != nil {
+				return err
+			}
+		}
+		_, err = decoder.Token()
+		return err
+	default:
+		return fmt.Errorf("unexpected GuideLLM delimiter %q", delimiter)
+	}
+}
+
+func decodeRawObject(raw json.RawMessage) map[string]json.RawMessage {
+	var object map[string]json.RawMessage
+	if json.Unmarshal(raw, &object) == nil {
+		return object
+	}
+	var encoded string
+	if json.Unmarshal(raw, &encoded) == nil && json.Unmarshal([]byte(encoded), &object) == nil {
+		return object
+	}
+	return nil
+}
+
+func rawString(raw json.RawMessage) (string, bool) {
+	var value string
+	if json.Unmarshal(raw, &value) != nil {
+		return "", false
+	}
+	return value, true
+}
+
+func extractPromptObject(object map[string]json.RawMessage) (string, bool) {
+	if object == nil {
+		return "", false
+	}
+	if prompt, ok := firstStringField(object, promptColumnCandidates); ok {
+		return prompt, true
+	}
+	for _, key := range []string{"body", "payload", "request"} {
+		if nested := decodeRawObject(object[key]); nested != nil {
+			if prompt, ok := extractPromptObject(nested); ok {
+				return prompt, true
+			}
+		}
+	}
+	var messages []map[string]json.RawMessage
+	if json.Unmarshal(object["messages"], &messages) != nil {
+		return "", false
+	}
+	userPrompt := ""
+	userCount := 0
+	for _, message := range messages {
+		role, _ := rawString(message["role"])
+		content, ok := rawString(message["content"])
+		if role == "user" && ok {
+			userPrompt = content
+			userCount++
+		}
+	}
+	return userPrompt, userCount == 1
+}
+
+func requestPrompt(object map[string]json.RawMessage) (string, bool) {
+	for _, key := range []string{"request_args", "request", "input"} {
+		if nested := decodeRawObject(object[key]); nested != nil {
+			if prompt, ok := extractPromptObject(nested); ok {
+				return prompt, true
+			}
+		}
+	}
+	return extractPromptObject(object)
+}
+
+func extractOutputRaw(raw json.RawMessage) (string, bool) {
+	object := decodeRawObject(raw)
+	if object == nil {
+		return rawString(raw)
+	}
+	for _, key := range []string{"content", "text", "output", "generated_text"} {
+		if value, ok := rawString(object[key]); ok {
+			return value, true
+		}
+	}
+	for _, key := range []string{"message", "body", "response"} {
+		if value, ok := extractOutputRaw(object[key]); ok {
+			return value, true
+		}
+	}
+	var choices []map[string]json.RawMessage
+	if json.Unmarshal(object["choices"], &choices) == nil {
+		for _, choice := range choices {
+			if value, ok := extractOutputRaw(choice["message"]); ok {
+				return value, true
+			}
+			if value, ok := rawString(choice["text"]); ok {
+				return value, true
+			}
+		}
+	}
+	return "", false
+}
+
+func requestOutput(object map[string]json.RawMessage) (string, bool) {
+	for _, key := range []string{"response", "response_args", "output", "result"} {
+		if value, ok := extractOutputRaw(object[key]); ok {
+			return value, true
+		}
+	}
+	return "", false
+}
+
+func requestID(object map[string]json.RawMessage, ordinal int) string {
+	for _, key := range []string{"id", "request_id", "request_uuid"} {
+		if value, ok := rawString(object[key]); ok && value != "" {
+			return value
+		}
+	}
+	return fmt.Sprintf("request-%d", ordinal)
+}
+
+func intPointer(value int) *int {
+	return &value
+}
+
+func scoreGuideRequest(
+	state string,
+	raw json.RawMessage,
+	index map[string]*datasetEntry,
+	ordinal int,
+) evaluationItem {
+	object := decodeRawObject(raw)
+	item := evaluationItem{RequestID: requestID(object, ordinal)}
+	prompt, ok := requestPrompt(object)
+	if !ok {
+		item.State = evaluationItemUnscored
+		item.Message = "single-turn prompt not found in request_args"
+		return item
+	}
+	promptExcerpt := makeEvaluationExcerpt(prompt)
+	item.PromptExcerpt = &promptExcerpt
+	entry := index[promptDigest(prompt)]
+	if entry == nil {
+		item.State = evaluationItemUnscored
+		item.Message = "prompt not found in evaluation dataset"
+		return item
+	}
+	item.DatasetIndex = intPointer(entry.index)
+	item.ReferenceExcerpt = &entry.answer.excerpt
+	if entry.conflict {
+		item.State = evaluationItemUnscored
+		item.Message = "duplicate prompt has conflicting reference answers"
+		return item
+	}
+	if !entry.answer.scorable {
+		item.State = evaluationItemUnscored
+		item.ReferenceExcerpt = nil
+		item.Message = "reference answer is missing or not a scalar"
+		return item
+	}
+	if state != "successful" {
+		item.State = evaluationItemIncorrect
+		item.Message = "request " + state
+		return item
+	}
+	output, ok := requestOutput(object)
+	if !ok || normalizeWhitespace(output) == "" {
+		item.State = evaluationItemIncorrect
+		item.Message = "successful request has empty output"
+		return item
+	}
+	outputExcerpt := makeEvaluationExcerpt(output)
+	item.OutputExcerpt = &outputExcerpt
+	matched, matchType := matchAnswer(entry.answer, output)
+	item.MatchType = matchType
+	if matched {
+		item.State = evaluationItemCorrect
+	} else {
+		item.State = evaluationItemIncorrect
+	}
+	return item
+}
+
+func addEvaluationItem(summary *api.LLMBenchmarkDatasetEvaluation, item evaluationItem) {
+	summary.RequestTotal++
+	switch item.State {
+	case evaluationItemCorrect:
+		summary.Correct++
+	case evaluationItemIncorrect:
+		summary.Incorrect++
+	case evaluationItemUnscored:
+		summary.Unscored++
+	}
+}
+
+var evaluationCSVHeader = []string{
+	"request_id", "dataset_index", "state", "match_type", "message",
+	"prompt_excerpt", "prompt_rune_count", "prompt_sha256", "prompt_truncated",
+	"reference_excerpt", "reference_rune_count", "reference_sha256", "reference_truncated",
+	"output_excerpt", "output_rune_count", "output_sha256", "output_truncated",
+}
+
+func excerptCSV(value *evaluationExcerpt) []string {
+	if value == nil {
+		return []string{"", "", "", ""}
+	}
+	return []string{
+		value.Text,
+		strconv.Itoa(value.RuneCount),
+		value.SHA256,
+		strconv.FormatBool(value.Truncated),
+	}
+}
+
+func evaluationCSVRow(item evaluationItem) []string {
+	datasetIndex := ""
+	if item.DatasetIndex != nil {
+		datasetIndex = strconv.Itoa(*item.DatasetIndex)
+	}
+	row := []string{item.RequestID, datasetIndex, item.State, item.MatchType, item.Message}
+	row = append(row, excerptCSV(item.PromptExcerpt)...)
+	row = append(row, excerptCSV(item.ReferenceExcerpt)...)
+	row = append(row, excerptCSV(item.OutputExcerpt)...)
+	return row
+}
+
+func EvaluateDataset(ctx context.Context, input EvaluationInput) (ret EvaluationResult, retErr error) {
+	if err := os.MkdirAll(input.OutputDir, 0755); err != nil {
+		return ret, err
+	}
+	ret.LogPath = filepath.Join(input.OutputDir, "evaluation.log")
+	logFile, err := os.Create(ret.LogPath)
+	if err != nil {
+		return ret, err
+	}
+	defer logFile.Close()
+	defer func() {
+		if retErr == nil {
+			return
+		}
+		ret.Summary.State = api.LLMBenchmarkEvaluationStateError
+		ret.Summary.Message = retErr.Error()
+		_, _ = fmt.Fprintf(logFile, "evaluation error: %s\n", retErr)
+	}()
+
+	column, skipMessage, err := detectAnswerColumn(ctx, input.DatasetPath, input.AnswerColumn)
+	if err != nil {
+		return ret, err
+	}
+	if skipMessage != "" {
+		ret.Summary.State = api.LLMBenchmarkEvaluationStateSkipped
+		ret.Summary.Message = skipMessage
+		_, _ = fmt.Fprintf(logFile, "evaluation skipped: %s\n", skipMessage)
+		return ret, nil
+	}
+	ret.Summary.AnswerColumn = column
+	_, _ = fmt.Fprintf(logFile, "answer column: %s\n", column)
+
+	index, err := buildDatasetIndex(ctx, input.DatasetPath, column)
+	if err != nil {
+		return ret, err
+	}
+
+	jsonPath := filepath.Join(input.OutputDir, "evaluation.json")
+	csvPath := filepath.Join(input.OutputDir, "evaluation.csv")
+	jsonFile, err := os.Create(jsonPath)
+	if err != nil {
+		return ret, err
+	}
+	csvFile, err := os.Create(csvPath)
+	if err != nil {
+		_ = jsonFile.Close()
+		_ = os.Remove(jsonPath)
+		return ret, err
+	}
+	reportsComplete := false
+	defer func() {
+		_ = jsonFile.Close()
+		_ = csvFile.Close()
+		if !reportsComplete {
+			_ = os.Remove(jsonPath)
+			_ = os.Remove(csvPath)
+		}
+	}()
+
+	if _, err := io.WriteString(jsonFile, "{\"items\":["); err != nil {
+		return ret, err
+	}
+	jsonEncoder := json.NewEncoder(jsonFile)
+	csvWriter := csv.NewWriter(csvFile)
+	if err := csvWriter.Write(evaluationCSVHeader); err != nil {
+		return ret, err
+	}
+	first := true
+	ordinal := 0
+	err = walkGuideLLMRequests(ctx, input.BenchmarkPath, func(state string, raw json.RawMessage) error {
+		ordinal++
+		item := scoreGuideRequest(state, raw, index, ordinal)
+		addEvaluationItem(&ret.Summary, item)
+		if !first {
+			if _, err := io.WriteString(jsonFile, ","); err != nil {
+				return err
+			}
+		}
+		first = false
+		if err := jsonEncoder.Encode(item); err != nil {
+			return err
+		}
+		return csvWriter.Write(evaluationCSVRow(item))
+	})
+	if err != nil {
+		return ret, err
+	}
+	if ret.Summary.RequestTotal == 0 {
+		err = errors.New("no GuideLLM request records found")
+		return ret, err
+	}
+
+	ret.Summary.Evaluated = ret.Summary.Correct + ret.Summary.Incorrect
+	if ret.Summary.Evaluated > 0 {
+		ret.Summary.Accuracy = float64(ret.Summary.Correct) / float64(ret.Summary.Evaluated)
+	} else {
+		ret.Summary.Message = "no scorable requests"
+	}
+	ret.Summary.State = api.LLMBenchmarkEvaluationStateCompleted
+	if _, err := io.WriteString(jsonFile, "],\"summary\":"); err != nil {
+		return ret, err
+	}
+	if err := jsonEncoder.Encode(ret.Summary); err != nil {
+		return ret, err
+	}
+	if _, err := io.WriteString(jsonFile, "}"); err != nil {
+		return ret, err
+	}
+	csvWriter.Flush()
+	if err := csvWriter.Error(); err != nil {
+		return ret, err
+	}
+	if err := jsonFile.Close(); err != nil {
+		return ret, err
+	}
+	if err := csvFile.Close(); err != nil {
+		return ret, err
+	}
+	reportsComplete = true
+	ret.ResultJSON = jsonPath
+	ret.ResultCSV = csvPath
+	_, _ = fmt.Fprintf(
+		logFile,
+		"requests=%d evaluated=%d correct=%d incorrect=%d unscored=%d accuracy=%g\n",
+		ret.Summary.RequestTotal,
+		ret.Summary.Evaluated,
+		ret.Summary.Correct,
+		ret.Summary.Incorrect,
+		ret.Summary.Unscored,
+		ret.Summary.Accuracy,
+	)
+	return ret, nil
+}

pkg/llm/benchmark/guidellm.go

@@ -0,0 +1,404 @@
+package benchmark
+
+import (
+	"encoding/csv"
+	"encoding/json"
+	"fmt"
+	"os"
+	"sort"
+	"strconv"
+	"strings"
+
+	commonapi "yunion.io/x/onecloud/pkg/apis"
+	computeapi "yunion.io/x/onecloud/pkg/apis/compute"
+)
+
+type GuideLLMSpecInput struct {
+	TargetURL           string
+	RequestFormat       string
+	Model               string
+	RequestRate         int
+	TotalRequests       int
+	MaxDurationSeconds  int
+	MaxErrors           int
+	DatasetInputTokens  int
+	DatasetOutputTokens int
+	DatasetPath         string
+}
+
+type GuideLLMSpec struct {
+	Backend     map[string]interface{}   `json:"backend"`
+	Profile     map[string]interface{}   `json:"profile"`
+	Tokenizer   map[string]interface{}   `json:"tokenizer,omitempty"`
+	Constraints []map[string]interface{} `json:"constraints"`
+	Data        []map[string]interface{} `json:"data"`
+}
+
+type LLMBenchmarkMetrics struct {
+	RequestsPerSecondMean *float64
+	RequestLatencyMeanSec *float64
+	RequestTotal          int
+	RequestSuccessful     int
+	RequestErrored        int
+	ErrorRate             *float64
+}
+
+type RunnerPodInput struct {
+	Name                   string
+	Image                  string
+	NetworkId              string
+	HostId                 string
+	CPU                    int
+	MemoryMB               int
+	PackageImageId         string
+	PackageSizeMB          int
+	PackageMountBase       string
+	ModelImageId           string
+	ModelSizeMB            int
+	ModelMountBase         string
+	ModelMountSubdirectory string
+}
+
+const DatasetPreflightSamples = 10
+
+func BuildGuideLLMSpec(input GuideLLMSpecInput) GuideLLMSpec {
+	backend := map[string]interface{}{
+		"kind":           "openai_http",
+		"target":         input.TargetURL,
+		"request_format": input.RequestFormat,
+	}
+	if input.Model != "" {
+		backend["model"] = input.Model
+	}
+	tokenizer := map[string]interface{}{}
+	if strings.TrimSpace(input.DatasetPath) != "" && input.Model != "" {
+		tokenizer = map[string]interface{}{
+			"kind":  "hf_auto",
+			"model": tokenizerModel(input.Model),
+		}
+	}
+	data := []map[string]interface{}{
+		{
+			"kind":          "synthetic_text",
+			"prompt_tokens": input.DatasetInputTokens,
+			"output_tokens": input.DatasetOutputTokens,
+		},
+	}
+	if strings.TrimSpace(input.DatasetPath) != "" {
+		data = []map[string]interface{}{
+			{
+				"kind": "json_file",
+				"path": input.DatasetPath,
+				"load_kwargs": map[string]interface{}{
+					"split": "train",
+				},
+			},
+		}
+	}
+	return GuideLLMSpec{
+		Backend: backend,
+		Profile: map[string]interface{}{
+			"kind": "constant",
+			"rate": input.RequestRate,
+		},
+		Tokenizer: tokenizer,
+		Constraints: []map[string]interface{}{
+			{"kind": "max_requests", "count": input.TotalRequests},
+			{"kind": "max_duration", "seconds": input.MaxDurationSeconds},
+			{"kind": "max_errors", "count": input.MaxErrors},
+		},
+		Data: data,
+	}
+}
+
+func GuideLLMLocalTokenizer(modelPath string) map[string]interface{} {
+	return map[string]interface{}{
+		"kind":  "hf_auto",
+		"model": modelPath,
+		"load_kwargs": map[string]interface{}{
+			"local_files_only": true,
+		},
+	}
+}
+
+func BuildGuideLLMPreflightSpec(formal GuideLLMSpec, maxDurationSeconds int) (GuideLLMSpec, error) {
+	if len(formal.Data) != 1 || formal.Data[0]["kind"] != "json_file" {
+		return GuideLLMSpec{}, fmt.Errorf("dataset preflight requires one json_file data source")
+	}
+	path, _ := formal.Data[0]["path"].(string)
+	if strings.TrimSpace(path) == "" {
+		return GuideLLMSpec{}, fmt.Errorf("dataset preflight path is empty")
+	}
+	data := map[string]interface{}{
+		"kind": "json_file",
+		"path": path,
+		"load_kwargs": map[string]interface{}{
+			"split": "train[:10]",
+		},
+	}
+	return GuideLLMSpec{
+		Backend:   formal.Backend,
+		Tokenizer: formal.Tokenizer,
+		Profile: map[string]interface{}{
+			"kind": "constant",
+			"rate": 1,
+		},
+		Constraints: []map[string]interface{}{
+			{"kind": "max_requests", "count": DatasetPreflightSamples},
+			{"kind": "max_duration", "seconds": maxDurationSeconds},
+			{"kind": "max_errors", "count": DatasetPreflightSamples},
+		},
+		Data: []map[string]interface{}{data},
+	}, nil
+}
+
+func GuideLLMRunCommand() []string {
+	return []string{
+		"sh",
+		"-lc",
+		"mkdir -p /workdir && guidellm run --output kind=json,path=/workdir/benchmarks.json --output kind=csv,path=/workdir/benchmarks.csv > /workdir/guidellm.log 2>&1",
+	}
+}
+
+func GuideLLMPreflightRunCommand(spec GuideLLMSpec) ([]string, error) {
+	envs, err := GuideLLMEnvs(spec)
+	if err != nil {
+		return nil, err
+	}
+	keys := make([]string, 0, len(envs))
+	for key := range envs {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+
+	parts := []string{"mkdir -p /workdir"}
+	for _, key := range keys {
+		parts = append(parts, "export "+key+"="+shellQuote(envs[key]))
+	}
+	parts = append(parts, "guidellm run --output kind=json,path=/workdir/dataset-preflight.json --output kind=csv,path=/workdir/dataset-preflight.csv > /workdir/dataset-preflight.log 2>&1")
+	return []string{"sh", "-lc", strings.Join(parts, " && ")}, nil
+}
+
+func shellQuote(value string) string {
+	return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'"
+}
+
+func BuildRunnerPodInput(input RunnerPodInput) *computeapi.ServerCreateInput {
+	root := int64(0)
+	ret := &computeapi.ServerCreateInput{
+		ServerConfigs: computeapi.NewServerConfigs(),
+		VcpuCount:     input.CPU,
+		VmemSize:      input.MemoryMB,
+		AutoStart:     true,
+		Pod: &computeapi.PodCreateInput{
+			Containers: []*computeapi.PodContainerCreateInput{
+				{
+					Name: "guidellm",
+					ContainerSpec: computeapi.ContainerSpec{
+						ContainerSpec: commonapi.ContainerSpec{
+							Image:         input.Image,
+							Command:       []string{"sh", "-lc", "while true; do sleep 3600; done"},
+							AlwaysRestart: false,
+							SecurityContext: &commonapi.ContainerSecurityContext{
+								RunAsUser:  &root,
+								RunAsGroup: &root,
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+	if input.PackageImageId != "" {
+		if input.PackageSizeMB <= 0 {
+			input.PackageSizeMB = 1024
+		}
+		if input.PackageMountBase == "" {
+			input.PackageMountBase = "/data/benchmark-packages"
+		}
+		diskIndex := len(ret.Disks)
+		ret.Disks = append(ret.Disks, &computeapi.DiskConfig{
+			DiskType: "data",
+			Format:   "raw",
+			Fs:       "ext4",
+			SizeMb:   input.PackageSizeMB,
+			Index:    diskIndex,
+		})
+		ret.Pod.Containers[0].VolumeMounts = append(ret.Pod.Containers[0].VolumeMounts, &commonapi.ContainerVolumeMount{
+			Type:      commonapi.CONTAINER_VOLUME_MOUNT_TYPE_DISK,
+			MountPath: input.PackageMountBase,
+			Disk: &commonapi.ContainerVolumeMountDisk{
+				Index:        &diskIndex,
+				SubDirectory: strings.TrimPrefix(input.PackageMountBase, "/"),
+				PostOverlay: []*commonapi.ContainerVolumeMountDiskPostOverlay{
+					{
+						Image: &commonapi.ContainerVolumeMountDiskPostImageOverlay{
+							Id: input.PackageImageId,
+						},
+					},
+				},
+			},
+			Propagation: commonapi.MOUNTPROPAGATION_PROPAGATION_HOST_TO_CONTAINER,
+		})
+	}
+	if input.ModelImageId != "" {
+		if input.ModelSizeMB <= 0 {
+			input.ModelSizeMB = 1024
+		}
+		if input.ModelMountBase == "" {
+			input.ModelMountBase = "/data/models"
+		}
+		diskIndex := len(ret.Disks)
+		ret.Disks = append(ret.Disks, &computeapi.DiskConfig{
+			DiskType: "data",
+			Format:   "raw",
+			Fs:       "ext4",
+			SizeMb:   input.ModelSizeMB,
+			Index:    diskIndex,
+		})
+		ret.Pod.Containers[0].VolumeMounts = append(ret.Pod.Containers[0].VolumeMounts, &commonapi.ContainerVolumeMount{
+			Type:      commonapi.CONTAINER_VOLUME_MOUNT_TYPE_DISK,
+			MountPath: input.ModelMountBase,
+			ReadOnly:  true,
+			Disk: &commonapi.ContainerVolumeMountDisk{
+				Index:        &diskIndex,
+				SubDirectory: input.ModelMountSubdirectory,
+				PostOverlay: []*commonapi.ContainerVolumeMountDiskPostOverlay{
+					{
+						Image: &commonapi.ContainerVolumeMountDiskPostImageOverlay{
+							Id: input.ModelImageId,
+						},
+					},
+				},
+			},
+			Propagation: commonapi.MOUNTPROPAGATION_PROPAGATION_HOST_TO_CONTAINER,
+		})
+	}
+	ret.Hypervisor = computeapi.HYPERVISOR_POD
+	ret.Name = input.Name
+	ret.PreferHost = input.HostId
+	ret.Count = 1
+	ret.Networks = []*computeapi.NetworkConfig{
+		{
+			Network: input.NetworkId,
+		},
+	}
+	return ret
+}
+
+func GuideLLMEnvs(spec GuideLLMSpec) (map[string]string, error) {
+	backend, err := json.Marshal(spec.Backend)
+	if err != nil {
+		return nil, err
+	}
+	profile, err := json.Marshal(spec.Profile)
+	if err != nil {
+		return nil, err
+	}
+	tokenizer, err := json.Marshal(spec.Tokenizer)
+	if err != nil {
+		return nil, err
+	}
+	constraints, err := json.Marshal(spec.Constraints)
+	if err != nil {
+		return nil, err
+	}
+	data, err := json.Marshal(spec.Data)
+	if err != nil {
+		return nil, err
+	}
+	return map[string]string{
+		"GUIDELLM__SPEC__BACKEND":     string(backend),
+		"GUIDELLM__SPEC__PROFILE":     string(profile),
+		"GUIDELLM__SPEC__TOKENIZER":   string(tokenizer),
+		"GUIDELLM__SPEC__CONSTRAINTS": string(constraints),
+		"GUIDELLM__SPEC__DATA":        string(data),
+	}, nil
+}
+
+func tokenizerModel(model string) string {
+	if i := strings.LastIndex(model, ":"); i > 0 {
+		return model[:i]
+	}
+	return model
+}
+
+func ParseMetricsCSV(path string) (*LLMBenchmarkMetrics, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	rows, err := csv.NewReader(f).ReadAll()
+	if err != nil {
+		return nil, err
+	}
+	if len(rows) >= 4 && len(rows[1]) > 0 && rows[1][0] == "Run ID" {
+		return parseGuideLLMv07MetricsCSV(rows), nil
+	}
+	if len(rows) < 2 {
+		return nil, fmt.Errorf("metrics csv has %d rows", len(rows))
+	}
+	values := map[string]string{}
+	for i, key := range rows[0] {
+		if i < len(rows[1]) {
+			values[key] = rows[1][i]
+		}
+	}
+	ret := &LLMBenchmarkMetrics{
+		RequestsPerSecondMean: parseFloatPtr(values["requests_per_second_mean"]),
+		RequestLatencyMeanSec: parseFloatPtr(values["request_latency_mean_sec"]),
+		RequestTotal:          parseInt(values["request_total"]),
+		RequestSuccessful:     parseInt(values["request_successful"]),
+		RequestErrored:        parseInt(values["request_errored"]),
+		ErrorRate:             parseFloatPtr(values["error_rate"]),
+	}
+	return ret, nil
+}
+
+func parseGuideLLMv07MetricsCSV(rows [][]string) *LLMBenchmarkMetrics {
+	value := func(group, name, unit string) string {
+		for i := range rows[1] {
+			if i < len(rows[0]) && i < len(rows[2]) && i < len(rows[3]) &&
+				rows[0][i] == group && rows[1][i] == name && (unit == "" || rows[2][i] == unit) {
+				return rows[3][i]
+			}
+		}
+		return ""
+	}
+	ret := &LLMBenchmarkMetrics{
+		RequestsPerSecondMean: parseFloatPtr(value("Server Throughput", "Successful Requests/Sec", "Mean")),
+		RequestLatencyMeanSec: parseFloatPtr(value("Scheduler Metrics", "Request Time Avg", "Sec")),
+		RequestTotal:          parseInt(value("Request Counts", "Total", "")),
+		RequestSuccessful:     parseInt(value("Request Counts", "Successful", "")),
+		RequestErrored:        parseInt(value("Request Counts", "Errored", "")),
+	}
+	if ret.RequestsPerSecondMean == nil {
+		if duration, err := strconv.ParseFloat(value("Timings", "Duration", "Sec"), 64); err == nil && duration > 0 {
+			v := float64(ret.RequestSuccessful) / duration
+			ret.RequestsPerSecondMean = &v
+		}
+	}
+	if ret.RequestTotal > 0 {
+		v := float64(ret.RequestErrored) / float64(ret.RequestTotal)
+		ret.ErrorRate = &v
+	}
+	return ret
+}
+
+func parseFloatPtr(s string) *float64 {
+	if s == "" {
+		return nil
+	}
+	v, err := strconv.ParseFloat(s, 64)
+	if err != nil {
+		return nil
+	}
+	return &v
+}
+
+func parseInt(s string) int {
+	v, _ := strconv.Atoi(s)
+	return v
+}

pkg/llm/models/llm_benchmark_package.go

@@ -0,0 +1,520 @@
+package models
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/httputils"
+	"yunion.io/x/sqlchemy"
+
+	commonapis "yunion.io/x/onecloud/pkg/apis"
+	imageapi "yunion.io/x/onecloud/pkg/apis/image"
+	api "yunion.io/x/onecloud/pkg/apis/llm"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/llm/options"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	imagemodules "yunion.io/x/onecloud/pkg/mcclient/modules/image"
+)
+
+func init() {
+	GetLLMBenchmarkPackageManager()
+}
+
+var llmBenchmarkPackageManager *SLLMBenchmarkPackageManager
+
+func GetLLMBenchmarkPackageManager() *SLLMBenchmarkPackageManager {
+	if llmBenchmarkPackageManager != nil {
+		return llmBenchmarkPackageManager
+	}
+	llmBenchmarkPackageManager = &SLLMBenchmarkPackageManager{
+		SSharableVirtualResourceBaseManager: db.NewSharableVirtualResourceBaseManager(
+			SLLMBenchmarkPackage{},
+			"llm_benchmark_packages_tbl",
+			"llm_benchmark_package",
+			"llm_benchmark_packages",
+		),
+	}
+	llmBenchmarkPackageManager.SetVirtualObject(llmBenchmarkPackageManager)
+	return llmBenchmarkPackageManager
+}
+
+type SLLMBenchmarkPackageManager struct {
+	db.SSharableVirtualResourceBaseManager
+}
+
+type SLLMBenchmarkPackage struct {
+	db.SSharableVirtualResourceBase
+
+	Source       string `width:"64" charset:"ascii" nullable:"false" default:"huggingface" list:"user" create:"optional"`
+	RepoId       string `width:"256" charset:"utf8" nullable:"true" list:"user" create:"optional"`
+	Revision     string `width:"128" charset:"utf8" nullable:"true" list:"user" create:"optional"`
+	FilePath     string `width:"512" charset:"utf8" nullable:"true" list:"user" create:"optional"`
+	Format       string `width:"64" charset:"ascii" nullable:"false" default:"guidellm_jsonl" list:"user" create:"optional"`
+	AnswerColumn string `width:"128" charset:"utf8" nullable:"true" list:"user" create:"optional"`
+
+	ImageId      string `width:"128" charset:"ascii" nullable:"true" list:"user" create:"optional"`
+	Size         int64  `nullable:"true" list:"user" create:"optional"`
+	ActualSizeMb int32  `nullable:"true" list:"user" create:"optional"`
+
+	MountPath   string `charset:"utf8" length:"medium" nullable:"true" list:"user" create:"optional"`
+	DatasetPath string `charset:"utf8" length:"medium" nullable:"true" list:"user" create:"optional"`
+	Manifest    string `charset:"utf8" length:"long" nullable:"true" list:"user" create:"optional"`
+}
+
+func (man *SLLMBenchmarkPackageManager) ValidateCreateData(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	ownerId mcclient.IIdentityProvider,
+	query jsonutils.JSONObject,
+	input api.LLMBenchmarkPackageCreateInput,
+) (api.LLMBenchmarkPackageCreateInput, error) {
+	var err error
+	input.SharableVirtualResourceCreateInput, err = man.SSharableVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.SharableVirtualResourceCreateInput)
+	if err != nil {
+		return input, errors.Wrap(err, "validate SharableVirtualResourceCreateInput")
+	}
+	defaultLLMBenchmarkPackageInput(&input)
+	if input.ImageId != "" {
+		img, err := fetchImage(ctx, userCred, input.ImageId)
+		if err != nil {
+			return input, errors.Wrap(err, "fetch image")
+		}
+		if img.DiskFormat != imageapi.IMAGE_DISK_FORMAT_TGZ {
+			return input, errors.Wrapf(httperrors.ErrInvalidFormat, "cannot use image of format %s", img.DiskFormat)
+		}
+		input.ImageId = img.Id
+		input.Size = img.Size
+		input.ActualSizeMb = img.MinDiskMB
+		input.Status = img.Status
+		return input, nil
+	}
+	if input.Source != api.LLMBenchmarkPackageSourceHuggingFace {
+		return input, errors.Wrap(httperrors.ErrInputParameter, "source only supports huggingface")
+	}
+	if input.Format != api.LLMBenchmarkPackageFormatGuideLLMJSONL {
+		return input, errors.Wrap(httperrors.ErrInputParameter, "format only supports guidellm_jsonl")
+	}
+	if strings.TrimSpace(input.RepoId) == "" {
+		return input, httperrors.NewMissingParameterError("repo_id")
+	}
+	if strings.TrimSpace(input.FilePath) == "" {
+		return input, httperrors.NewMissingParameterError("file_path")
+	}
+	input.Status = imageapi.IMAGE_STATUS_QUEUED
+	return input, nil
+}
+
+func defaultLLMBenchmarkPackageInput(input *api.LLMBenchmarkPackageCreateInput) {
+	input.AnswerColumn = strings.TrimSpace(input.AnswerColumn)
+	if input.Source == "" {
+		input.Source = api.LLMBenchmarkPackageSourceHuggingFace
+	}
+	if input.Revision == "" {
+		input.Revision = "main"
+	}
+	if input.Format == "" {
+		input.Format = api.LLMBenchmarkPackageFormatGuideLLMJSONL
+	}
+	if input.MountPath == "" {
+		input.MountPath = path.Join(api.LLMBenchmarkPackageMountBase, benchmarkPackagePathName(input.Name))
+	}
+	if input.DatasetPath == "" {
+		input.DatasetPath = path.Join(input.MountPath, "data.jsonl")
+	}
+}
+
+func benchmarkPackagePathName(name string) string {
+	name = strings.TrimSpace(name)
+	var b strings.Builder
+	lastDash := false
+	for _, r := range name {
+		keep := (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-'
+		if keep {
+			b.WriteRune(r)
+			lastDash = false
+			continue
+		}
+		if !lastDash {
+			b.WriteByte('-')
+			lastDash = true
+		}
+	}
+	out := strings.Trim(b.String(), "-")
+	if out == "" {
+		return "package"
+	}
+	return out
+}
+
+func (man *SLLMBenchmarkPackageManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, input api.LLMBenchmarkPackageListInput) (*sqlchemy.SQuery, error) {
+	q, err := man.SSharableVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, input.SharableVirtualResourceListInput)
+	if err != nil {
+		return nil, errors.Wrap(err, "ListItemFilter")
+	}
+	if input.Source != "" {
+		q = q.Equals("source", input.Source)
+	}
+	if input.RepoId != "" {
+		q = q.Equals("repo_id", input.RepoId)
+	}
+	if input.Format != "" {
+		q = q.Equals("format", input.Format)
+	}
+	return q, nil
+}
+
+func (man *SLLMBenchmarkPackageManager) PerformImport(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	input api.LLMBenchmarkPackageImportInput,
+) (*SLLMBenchmarkPackage, error) {
+	input.ImageId = ""
+	data := jsonutils.Marshal(input)
+	obj, err := db.DoCreate(man, ctx, userCred, nil, data, userCred)
+	if err != nil {
+		return nil, errors.Wrap(err, "DoCreate")
+	}
+	pkg := obj.(*SLLMBenchmarkPackage)
+	if err := data.Unmarshal(&input); err != nil {
+		_ = pkg.SetStatus(ctx, userCred, imageapi.IMAGE_STATUS_KILLED, err.Error())
+		return nil, errors.Wrap(err, "unmarshal validated import input")
+	}
+	if err := pkg.startImportTask(ctx, userCred, input, ""); err != nil {
+		_ = pkg.SetStatus(ctx, userCred, imageapi.IMAGE_STATUS_KILLED, err.Error())
+		return nil, err
+	}
+	return pkg, nil
+}
+
+func (pkg *SLLMBenchmarkPackage) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) {
+	pkg.SSharableVirtualResourceBase.PostCreate(ctx, userCred, ownerId, query, data)
+	if pkg.ImageId != "" {
+		return
+	}
+	input := api.LLMBenchmarkPackageImportInput{}
+	if err := data.Unmarshal(&input); err != nil {
+		_ = pkg.SetStatus(ctx, userCred, imageapi.IMAGE_STATUS_KILLED, err.Error())
+		return
+	}
+	if err := pkg.startImportTask(ctx, userCred, input, ""); err != nil {
+		_ = pkg.SetStatus(ctx, userCred, imageapi.IMAGE_STATUS_KILLED, err.Error())
+	}
+}
+
+func (pkg *SLLMBenchmarkPackage) startImportTask(ctx context.Context, userCred mcclient.TokenCredential, input api.LLMBenchmarkPackageImportInput, parentTaskId string) error {
+	params := jsonutils.NewDict()
+	params.Add(jsonutils.Marshal(input), "import_input")
+	task, err := taskman.TaskManager.NewTask(ctx, "LLMBenchmarkPackageImportTask", pkg, userCred, params, parentTaskId, "")
+	if err != nil {
+		return errors.Wrap(err, "NewTask")
+	}
+	return task.ScheduleRun(nil)
+}
+
+func (pkg *SLLMBenchmarkPackage) updateImportStatus(ctx context.Context, userCred mcclient.TokenCredential, status, reason string) error {
+	if pkg.Status == status {
+		return nil
+	}
+	_, err := db.Update(pkg, func() error {
+		pkg.Status = status
+		return nil
+	})
+	return errors.Wrap(err, reason)
+}
+
+func (pkg *SLLMBenchmarkPackage) DoImport(ctx context.Context, userCred mcclient.TokenCredential, s *mcclient.ClientSession, input api.LLMBenchmarkPackageCreateInput) (string, error) {
+	_ = pkg.SetProgress(0)
+	tmpDir, err := getLLMBenchmarkPackageImportWorkDir(options.Options.LLMWorkingDirectory, input)
+	if err != nil {
+		return "", err
+	}
+	if err := os.MkdirAll(tmpDir, 0755); err != nil {
+		return "", errors.Wrap(err, "mkdir import work dir")
+	}
+	rootDir := filepath.Join(tmpDir, "package")
+	dataDir := filepath.Join(rootDir, "data")
+	if err := os.MkdirAll(dataDir, 0755); err != nil {
+		return tmpDir, errors.Wrap(err, "mkdir data dir")
+	}
+	if err := downloadHuggingFaceBenchmarkPackageFile(ctx, filepath.Join(dataDir, "data.jsonl"), input); err != nil {
+		return tmpDir, errors.Wrap(err, "download benchmark package")
+	}
+	_ = pkg.SetProgress(90)
+
+	manifest := map[string]string{
+		"source":        input.Source,
+		"repo_id":       input.RepoId,
+		"revision":      input.Revision,
+		"file_path":     input.FilePath,
+		"format":        input.Format,
+		"dataset_path":  pkg.DatasetPath,
+		"answer_column": input.AnswerColumn,
+	}
+	manifestStr := jsonutils.Marshal(manifest).String()
+	if err := os.WriteFile(filepath.Join(rootDir, "manifest.json"), []byte(manifestStr), 0644); err != nil {
+		return tmpDir, errors.Wrap(err, "write manifest")
+	}
+	archivePath := filepath.Join(tmpDir, "package.tgz")
+	if err := createTarGz(rootDir, archivePath); err != nil {
+		return tmpDir, errors.Wrap(err, "createTarGz")
+	}
+	_ = pkg.SetProgress(95)
+
+	if err := pkg.updateImportStatus(ctx, userCred, imageapi.IMAGE_STATUS_SAVING, "saving"); err != nil {
+		return tmpDir, err
+	}
+	imageId, imgSize, err := pkg.uploadImage(ctx, userCred, s, input, archivePath, manifestStr)
+	if err != nil {
+		return tmpDir, errors.Wrap(err, "upload image")
+	}
+	_ = pkg.SetProgress(98)
+
+	_, err = db.Update(pkg, func() error {
+		pkg.ImageId = imageId
+		pkg.Size = imgSize
+		pkg.Manifest = manifestStr
+		return nil
+	})
+	if err != nil {
+		return tmpDir, errors.Wrap(err, "update benchmark package")
+	}
+	if _, err := pkg.WaitImageStatus(ctx, userCred, []string{imageapi.IMAGE_STATUS_ACTIVE}, 1800); err != nil {
+		return tmpDir, errors.Wrap(err, "wait image")
+	}
+	if err := pkg.syncImageStatus(ctx, userCred); err != nil {
+		return tmpDir, errors.Wrap(err, "sync image")
+	}
+	_ = pkg.SetProgress(100)
+	return tmpDir, nil
+}
+
+func getLLMBenchmarkPackageImportWorkDir(root string, input api.LLMBenchmarkPackageCreateInput) (string, error) {
+	root = strings.TrimSpace(root)
+	if root == "" {
+		return "", errors.Error("LLMWorkingDirectory is empty")
+	}
+	key := strings.Join([]string{input.Source, input.RepoId, input.Revision, input.FilePath, input.Format, input.AnswerColumn}, "\x00")
+	sum := sha256.Sum256([]byte(key))
+	display := benchmarkPackagePathName(input.Name)
+	return filepath.Join(root, "benchmark-package-import-cache", fmt.Sprintf("%s-%s", display, hex.EncodeToString(sum[:])[:16])), nil
+}
+
+func (pkg *SLLMBenchmarkPackage) uploadImage(ctx context.Context, userCred mcclient.TokenCredential, s *mcclient.ClientSession, input api.LLMBenchmarkPackageCreateInput, archivePath string, manifest string) (string, int64, error) {
+	f, err := os.Open(archivePath)
+	if err != nil {
+		return "", 0, errors.Wrap(err, "open archive")
+	}
+	defer f.Close()
+	stat, err := f.Stat()
+	if err != nil {
+		return "", 0, errors.Wrap(err, "stat archive")
+	}
+	size := stat.Size()
+	protected := false
+	imgParams := imageapi.ImageCreateInput{}
+	imgParams.GenerateName = pkg.Name
+	imgParams.DiskFormat = imageapi.IMAGE_DISK_FORMAT_TGZ
+	imgParams.Size = &size
+	imgParams.Protected = &protected
+	imgParams.Properties = map[string]string{
+		"llm_benchmark_package":             "true",
+		"source":                            input.Source,
+		"source_repo_id":                    input.RepoId,
+		"source_requested_revision":         input.Revision,
+		"source_file_path":                  input.FilePath,
+		"format":                            input.Format,
+		"answer_column":                     input.AnswerColumn,
+		"manifest":                          manifest,
+		imageapi.IMAGE_INTERNAL_PATH_MAP:    jsonutils.Marshal(map[string]string{"data": pkg.MountPath}).String(),
+		imageapi.IMAGE_USED_BY_POST_OVERLAY: "true",
+	}
+	imageObj, err := imagemodules.Images.Upload(s, jsonutils.Marshal(imgParams), f, size)
+	if err != nil {
+		return "", 0, errors.Wrap(err, "Upload")
+	}
+	imageId, err := imageObj.GetString("id")
+	if err != nil {
+		return "", 0, errors.Wrap(err, "GetString id")
+	}
+	return imageId, size, nil
+}
+
+func (pkg *SLLMBenchmarkPackage) syncImageStatus(ctx context.Context, userCred mcclient.TokenCredential) error {
+	img, err := fetchImage(ctx, userCred, pkg.ImageId)
+	if err != nil {
+		return err
+	}
+	_, err = db.Update(pkg, func() error {
+		pkg.Status = img.Status
+		pkg.Size = img.Size
+		pkg.ActualSizeMb = img.MinDiskMB
+		return nil
+	})
+	return err
+}
+
+func (pkg *SLLMBenchmarkPackage) WaitImageStatus(ctx context.Context, userCred mcclient.TokenCredential, targetStatus []string, timeoutSecs int) (*imageapi.ImageDetails, error) {
+	expire := time.Now().Add(time.Second * time.Duration(timeoutSecs))
+	for time.Now().Before(expire) {
+		img, err := fetchImage(ctx, userCred, pkg.ImageId)
+		if err != nil {
+			return nil, err
+		}
+		for _, status := range targetStatus {
+			if img.Status == status {
+				return img, nil
+			}
+		}
+		if img.Status == imageapi.IMAGE_STATUS_KILLED || img.Status == imageapi.IMAGE_STATUS_DEACTIVATED {
+			return nil, errors.Wrap(errors.ErrInvalidStatus, img.Status)
+		}
+		time.Sleep(2 * time.Second)
+	}
+	return nil, errors.Wrapf(httperrors.ErrTimeout, "wait image status %s timeout", targetStatus)
+}
+
+func (pkg *SLLMBenchmarkPackage) CleanupImportTmpDir(ctx context.Context, userCred mcclient.TokenCredential, dir string) {
+	if dir == "" {
+		return
+	}
+	if err := os.RemoveAll(dir); err != nil {
+		log.Warningf("cleanup benchmark package import dir %s: %s", dir, err)
+	}
+}
+
+func downloadHuggingFaceBenchmarkPackageFile(ctx context.Context, dst string, input api.LLMBenchmarkPackageCreateInput) error {
+	endpoint := strings.TrimRight(options.Options.HuggingFaceEndpoint, "/")
+	if endpoint == "" {
+		endpoint = huggingFaceMirrorEndpoint
+	}
+	fileURL := buildHuggingFaceDatasetFileURL(endpoint, input.RepoId, input.Revision, input.FilePath)
+	if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
+		return err
+	}
+	tmp := dst + ".tmp"
+	f, err := os.Create(tmp)
+	if err != nil {
+		return errors.Wrap(err, "create tmp")
+	}
+	client := httputils.GetTimeoutClient(0)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, fileURL, nil)
+	if err != nil {
+		_ = f.Close()
+		return err
+	}
+	if options.Options.HuggingFaceToken != "" {
+		req.Header.Set("Authorization", "Bearer "+options.Options.HuggingFaceToken)
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		_ = f.Close()
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		_ = f.Close()
+		return errors.Errorf("download %s failed: %s", fileURL, resp.Status)
+	}
+	if _, err := io.Copy(f, resp.Body); err != nil {
+		_ = f.Close()
+		return err
+	}
+	if err := f.Close(); err != nil {
+		return err
+	}
+	return os.Rename(tmp, dst)
+}
+
+func buildHuggingFaceDatasetFileURL(endpoint, repoID, revision, filePath string) string {
+	return fmt.Sprintf("%s/datasets/%s/resolve/%s/%s",
+		strings.TrimRight(endpoint, "/"),
+		escapeURLPathPreserveSlash(repoID),
+		url.PathEscape(revision),
+		escapeURLPathPreserveSlash(filePath),
+	)
+}
+
+func validateBenchmarkPackageDeleteStatus(status string) error {
+	if status == imageapi.IMAGE_STATUS_SAVING || status == imageapi.IMAGE_STATUS_QUEUED || status == commonapis.STATUS_DELETING {
+		return httperrors.NewInvalidStatusError("benchmark package is %s", status)
+	}
+	return nil
+}
+
+func (pkg *SLLMBenchmarkPackage) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error {
+	if err := validateBenchmarkPackageDeleteStatus(pkg.Status); err != nil {
+		return err
+	}
+	if err := ValidateBenchmarkPackageUnused(pkg.Id, ""); err != nil {
+		return err
+	}
+	return pkg.SSharableVirtualResourceBase.ValidateDeleteCondition(ctx, info)
+}
+
+func (pkg *SLLMBenchmarkPackage) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
+	manager := GetLLMBenchmarkManager()
+	lockKey := db.GetLockClassKey(manager, pkg.GetOwnerId())
+	lockman.LockClass(ctx, manager, lockKey)
+	defer lockman.ReleaseClass(ctx, manager, lockKey)
+
+	if err := ValidateBenchmarkPackageUnused(pkg.Id, ""); err != nil {
+		return err
+	}
+	return pkg.StartDeleteTask(ctx, userCred, query, "")
+}
+
+func (pkg *SLLMBenchmarkPackage) StartDeleteTask(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, parentTaskId string) error {
+	if err := validateBenchmarkPackageDeleteStatus(pkg.Status); err != nil {
+		return err
+	}
+	previousStatus := pkg.Status
+	if err := pkg.SetStatus(ctx, userCred, commonapis.STATUS_DELETING, ""); err != nil {
+		return errors.Wrap(err, "set benchmark package deleting")
+	}
+	rollbackStatus := func(startErr error) error {
+		if err := pkg.SetStatus(ctx, userCred, previousStatus, startErr.Error()); err != nil {
+			return errors.Wrapf(startErr, "rollback benchmark package status: %s", err)
+		}
+		return startErr
+	}
+	params := jsonutils.NewDict()
+	if pkg.ImageId != "" {
+		params.Set("image_id", jsonutils.NewString(pkg.ImageId))
+	}
+	if jsonutils.QueryBoolean(query, "purge", false) {
+		params.Set("purge", jsonutils.JSONTrue)
+	}
+	task, err := taskman.TaskManager.NewTask(ctx, "LLMBenchmarkPackageDeleteTask", pkg, userCred, params, parentTaskId, "")
+	if err != nil {
+		return rollbackStatus(errors.Wrap(err, "NewTask LLMBenchmarkPackageDeleteTask"))
+	}
+	if err := task.ScheduleRun(nil); err != nil {
+		return rollbackStatus(errors.Wrap(err, "ScheduleRun LLMBenchmarkPackageDeleteTask"))
+	}
+	return nil
+}
+
+func (pkg *SLLMBenchmarkPackage) Delete(ctx context.Context, userCred mcclient.TokenCredential) error {
+	return nil
+}
+
+func (pkg *SLLMBenchmarkPackage) RealDelete(ctx context.Context, userCred mcclient.TokenCredential) error {
+	return pkg.SSharableVirtualResourceBase.Delete(ctx, userCred)
+}

pkg/llm/models/llm_benchmark_stop_state.go

@@ -0,0 +1,13 @@
+package models
+
+import api "yunion.io/x/onecloud/pkg/apis/llm"
+
+func benchmarkRunFinalState(state string, stopRequested bool, runErr error) (string, string) {
+	if stopRequested || state == api.LLMBenchmarkStateStopped {
+		return api.LLMBenchmarkStateStopped, ""
+	}
+	if runErr != nil {
+		return api.LLMBenchmarkStateError, runErr.Error()
+	}
+	return api.LLMBenchmarkStateCompleted, ""
+}

pkg/llm/options/option.go

@@ -29,6 +29,24 @@ type LLMOptions struct {
 	ImportTaskWorkerCount int `help:"import task worker count" default:"8"`
 	StartTaskWorkerCount  int `help:"start task worker count" default:"128"`
 
+	LLMBenchmarkWorkDir              string `help:"llm benchmark working directory" default:"/opt/cloud/workspace/llm/benchmarks"`
+	LLMBenchmarkDefaultImage         string `help:"default GuideLLM benchmark image" default:"registry.cn-beijing.aliyuncs.com/cloudpods/guidellm:v0.7.0-amd64"`
+	LLMBenchmarkRunnerCPU            int    `help:"llm benchmark runner cpu" default:"1"`
+	LLMBenchmarkRunnerMemoryMB       int    `help:"llm benchmark runner memory MB" default:"2048"`
+	LLMBenchmarkDefaultRequestRate   int    `help:"default benchmark request rate" default:"1"`
+	LLMBenchmarkDefaultTotalRequests int    `help:"default benchmark total requests" default:"100"`
+	LLMBenchmarkDefaultInputTokens   int    `help:"default synthetic prompt tokens" default:"1024"`
+	LLMBenchmarkDefaultOutputTokens  int    `help:"default synthetic output tokens" default:"128"`
+	LLMBenchmarkMaxDurationSeconds   int    `help:"max benchmark duration seconds" default:"3600"`
+	LLMBenchmarkMaxRequestRate       int    `help:"max benchmark request rate" default:"100"`
+	LLMBenchmarkMaxTotalRequests     int    `help:"max benchmark total requests" default:"100000"`
+	ArtifactS3Endpoint               string `help:"MinIO/S3 endpoint for benchmark artifacts; empty disables upload" default:"http://monitor-minio.onecloud-monitoring.svc:9000"`
+	ArtifactS3AccessKey              string `help:"MinIO/S3 access key for benchmark artifacts" default:"monitor-admin"`
+	ArtifactS3SecretKey              string `help:"MinIO/S3 secret key for benchmark artifacts" default:"monitor-admin"`
+	ArtifactS3Bucket                 string `help:"MinIO/S3 bucket for benchmark artifacts" default:"llm-benchmark"`
+	ArtifactS3Secure                 bool   `help:"Use HTTPS for benchmark artifact endpoint without scheme" default:"false"`
+	ArtifactS3Prefix                 string `help:"MinIO/S3 object key prefix for benchmark artifacts" default:"llm-benchmarks"`
+
 	// MCP Agent 配置
 	MCPServerURL    string `help:"MCP Server URL" default:"http://default-mcp-server:30876"`
 	MCPAgentTimeout int    `help:"MCP Agent request timeout in seconds" default:"120"`

pkg/llm/service/benchmark_handler.go

@@ -0,0 +1,214 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/onecloud/pkg/appsrv"
+	"yunion.io/x/onecloud/pkg/cloudcommon/policy"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	bench "yunion.io/x/onecloud/pkg/llm/benchmark"
+	"yunion.io/x/onecloud/pkg/llm/models"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+)
+
+func AddBenchmarkArtifactHandlers(app *appsrv.Application) {
+	app.AddHandler2("GET", "/llm_benchmarks/<id>/artifacts", auth.Authenticate(handleLLMBenchmarkArtifacts), nil, "llm_benchmark_artifacts", nil)
+	app.AddHandler2("GET", "/llm_benchmarks/<id>/artifacts/<type>", auth.Authenticate(handleLLMBenchmarkArtifact), nil, "llm_benchmark_artifact", nil)
+}
+
+type benchmarkArtifactItem struct {
+	Type     string `json:"type"`
+	Filename string `json:"filename"`
+	Ready    bool   `json:"ready"`
+}
+
+func handleLLMBenchmarkArtifacts(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	b, ok := fetchLLMBenchmarkForArtifact(ctx, w, benchmarkArtifactParam(ctx, "<id>"))
+	if !ok {
+		return
+	}
+	items, err := buildBenchmarkArtifactList(ctx, b, bench.DefaultArtifactStore())
+	if err != nil {
+		httperrors.GeneralServerError(ctx, w, err)
+		return
+	}
+	appsrv.SendJSON(w, jsonutils.Marshal(map[string]interface{}{
+		"artifacts": items,
+	}))
+}
+
+func handleLLMBenchmarkArtifact(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	id := benchmarkArtifactParam(ctx, "<id>")
+	kind := benchmarkArtifactParam(ctx, "<type>")
+	if kind == "" {
+		httperrors.InvalidInputError(ctx, w, "missing artifact type")
+		return
+	}
+	sendBenchmarkArtifact(ctx, w, r, id, kind)
+}
+
+func sendBenchmarkArtifact(ctx context.Context, w http.ResponseWriter, r *http.Request, id, kind string) {
+	b, ok := fetchLLMBenchmarkForArtifact(ctx, w, id)
+	if !ok {
+		return
+	}
+	found, err := writeBenchmarkArtifact(ctx, w, r, b, kind, bench.DefaultArtifactStore())
+	if err != nil {
+		httperrors.GeneralServerError(ctx, w, err)
+		return
+	}
+	if !found {
+		httperrors.NotFoundError(ctx, w, "artifact %s not found", kind)
+	}
+}
+
+func setBenchmarkArtifactHeaders(w http.ResponseWriter, kind string) {
+	w.Header().Set("Content-Type", artifactContentType(kind))
+	w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", artifactFilename(kind)))
+}
+
+func writeBenchmarkArtifact(
+	ctx context.Context,
+	w http.ResponseWriter,
+	r *http.Request,
+	b *models.SLLMBenchmark,
+	kind string,
+	store *bench.ArtifactStore,
+) (bool, error) {
+	location, err := b.ArtifactPath(kind)
+	if err != nil {
+		return false, err
+	}
+	if location != "" && !strings.HasPrefix(location, "s3://") {
+		if ready, err := store.Exists(ctx, location); err != nil {
+			return false, err
+		} else if ready {
+			setBenchmarkArtifactHeaders(w, kind)
+			http.ServeFile(w, r, location)
+			return true, nil
+		}
+	}
+	if strings.HasPrefix(location, "s3://") {
+		body, err := store.Open(ctx, location)
+		if err != nil {
+			return false, err
+		}
+		defer body.Close()
+		setBenchmarkArtifactHeaders(w, kind)
+		_, err = io.Copy(w, body)
+		return true, err
+	}
+	raw := artifactRaw(b, kind)
+	if raw == "" {
+		return false, nil
+	}
+	setBenchmarkArtifactHeaders(w, kind)
+	_, err = io.WriteString(w, raw)
+	return true, err
+}
+
+func benchmarkArtifactParam(ctx context.Context, key string) string {
+	params := appsrv.AppContextGetParams(ctx)
+	return params.Params[key]
+}
+
+func fetchLLMBenchmarkForArtifact(ctx context.Context, w http.ResponseWriter, id string) (*models.SLLMBenchmark, bool) {
+	userCred := auth.FetchUserCredential(ctx, policy.FilterPolicyCredential)
+	if userCred == nil {
+		httperrors.UnauthorizedError(ctx, w, "Unauthorized")
+		return nil, false
+	}
+	obj, err := models.GetLLMBenchmarkManager().FetchByIdOrName(ctx, userCred, id)
+	if err != nil {
+		httperrors.GeneralServerError(ctx, w, err)
+		return nil, false
+	}
+	return obj.(*models.SLLMBenchmark), true
+}
+
+func buildBenchmarkArtifactList(
+	ctx context.Context,
+	b *models.SLLMBenchmark,
+	store *bench.ArtifactStore,
+) ([]benchmarkArtifactItem, error) {
+	kinds := []string{
+		"preflight", "preflight-log", "log", "json", "csv",
+		"evaluation", "evaluation-csv", "evaluation-log",
+	}
+	ret := make([]benchmarkArtifactItem, 0, len(kinds))
+	for _, kind := range kinds {
+		location, err := b.ArtifactPath(kind)
+		if err != nil {
+			return nil, err
+		}
+		ready, err := store.Exists(ctx, location)
+		if err != nil {
+			return nil, err
+		}
+		if !ready {
+			ready = artifactRaw(b, kind) != ""
+		}
+		ret = append(ret, benchmarkArtifactItem{
+			Type:     kind,
+			Filename: artifactFilename(kind),
+			Ready:    ready,
+		})
+	}
+	return ret, nil
+}
+
+func artifactRaw(b *models.SLLMBenchmark, kind string) string {
+	switch kind {
+	case "preflight":
+		return b.RawPreflightResult
+	case "preflight-log":
+		return b.RawPreflightLog
+	case "log":
+		return b.RawLog
+	case "json":
+		return b.RawMetrics
+	case "csv":
+		return b.RawCsv
+	default:
+		return ""
+	}
+}
+
+func artifactFilename(kind string) string {
+	switch kind {
+	case "preflight":
+		return "dataset-preflight.json"
+	case "preflight-log":
+		return "dataset-preflight.log"
+	case "log":
+		return "guidellm.log"
+	case "json":
+		return "benchmarks.json"
+	case "csv":
+		return "benchmarks.csv"
+	case "evaluation":
+		return "evaluation.json"
+	case "evaluation-csv":
+		return "evaluation.csv"
+	case "evaluation-log":
+		return "evaluation.log"
+	default:
+		return kind
+	}
+}
+
+func artifactContentType(kind string) string {
+	switch kind {
+	case "json", "preflight", "evaluation":
+		return "application/json"
+	case "csv", "evaluation-csv":
+		return "text/csv"
+	default:
+		return "text/plain; charset=utf-8"
+	}
+}

pkg/llm/service/handler.go

@@ -328,6 +328,7 @@ func InitHandlers(app *appsrv.Application, isSlave bool) {
 	app.AddHandler2("GET", "/llm_images_catalogs/<id>", auth.Authenticate(handleLLMImagesCatalogShow), nil, "llm_images_catalog_show", nil)
 
 	AddAvailableNetworkHandler(models.GetLLMManager().KeywordPlural(), app)
+	AddBenchmarkArtifactHandlers(app)
 
 	// 默认 Agent 聊天流：优先于 dispatcher 注册，避免被 performClassAction 的 sendJSON 覆盖。
 	// 注册两种路径：default-chat-stream（apigateway 转发用）与 default/chat-stream（climc 直连 region 时用，否则会被当作 resid=default 的 perform 导致 404）
@@ -365,6 +366,8 @@ func InitHandlers(app *appsrv.Application, isSlave bool) {
 		models.GetLLMContainerManager(),
 		models.GetLLMDeploymentManager(),
 		models.GetLLMManager(),
+		models.GetLLMBenchmarkManager(),
+		models.GetLLMBenchmarkPackageManager(),
 		// models.GetDifyManager(),
 		models.GetInstantModelManager(),
 		models.GetLLMInstantModelManager(),

pkg/llm/service/service.go

@@ -13,6 +13,7 @@ import (
 	app_common "yunion.io/x/onecloud/pkg/cloudcommon/app"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
+	bench "yunion.io/x/onecloud/pkg/llm/benchmark"
 	_ "yunion.io/x/onecloud/pkg/llm/drivers/llm_client"
 	_ "yunion.io/x/onecloud/pkg/llm/drivers/llm_container"
 	"yunion.io/x/onecloud/pkg/llm/models"
@@ -28,6 +29,14 @@ func StartService() {
 	dbOpts := &options.Options.DBOptions
 	baseOpts := &opts.BaseOptions
 	common_options.ParseOptions(opts, os.Args, "llm.conf", api.SERVICE_TYPE)
+	bench.ConfigureArtifactStore(bench.ArtifactStoreOptions{
+		Endpoint:  opts.ArtifactS3Endpoint,
+		AccessKey: opts.ArtifactS3AccessKey,
+		SecretKey: opts.ArtifactS3SecretKey,
+		Bucket:    opts.ArtifactS3Bucket,
+		Secure:    opts.ArtifactS3Secure,
+		Prefix:    opts.ArtifactS3Prefix,
+	})
 
 	llmTask.InitInstantModelSyncTaskManager()
 	app_common.InitAuth(commonOpts, func() {

pkg/llm/tasks/llm/llm_benchmark_delete_task.go

@@ -0,0 +1,96 @@
+package llm
+
+import (
+	"context"
+	"database/sql"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/llm/models"
+	"yunion.io/x/onecloud/pkg/util/logclient"
+)
+
+type LLMBenchmarkDeleteTask struct {
+	taskman.STask
+}
+
+func init() {
+	taskman.RegisterTask(LLMBenchmarkDeleteTask{})
+}
+
+func (task *LLMBenchmarkDeleteTask) taskFailed(ctx context.Context, benchmark *models.SLLMBenchmark, err error) {
+	_ = benchmark.SetState(ctx, task.UserCred, benchmark.State, err.Error())
+	db.OpsLog.LogEvent(benchmark, db.ACT_DELETE_FAIL, err, task.UserCred)
+	logclient.AddActionLogWithStartable(task, benchmark, logclient.ACT_DELETE, err, task.UserCred, false)
+	task.SetStageFailed(ctx, jsonutils.NewString(err.Error()))
+}
+
+func (task *LLMBenchmarkDeleteTask) deleteBenchmark(ctx context.Context, benchmark *models.SLLMBenchmark) {
+	if err := benchmark.CleanupArtifacts(ctx); err != nil {
+		log.Warningf("cleanup benchmark %s artifacts during delete: %s", benchmark.Id, err)
+	}
+	if err := benchmark.RealDelete(ctx, task.UserCred); err != nil {
+		task.taskFailed(ctx, benchmark, err)
+		return
+	}
+	logclient.AddActionLogWithStartable(task, benchmark, logclient.ACT_DELETE, nil, task.UserCred, true)
+	task.SetStageComplete(ctx, nil)
+}
+
+func benchmarkPackageDeleteRequired(otherReferences int) bool {
+	return otherReferences == 0
+}
+
+func (task *LLMBenchmarkDeleteTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
+	benchmark := obj.(*models.SLLMBenchmark)
+	packageID := benchmark.BenchmarkPackageId
+	if packageID == "" {
+		task.deleteBenchmark(ctx, benchmark)
+		return
+	}
+
+	manager := models.GetLLMBenchmarkManager()
+	lockKey := db.GetLockClassKey(manager, benchmark.GetOwnerId())
+	lockman.LockClass(ctx, manager, lockKey)
+	defer lockman.ReleaseClass(ctx, manager, lockKey)
+
+	otherReferences, err := models.CountBenchmarkPackageReferences(packageID, benchmark.Id)
+	if err != nil {
+		task.taskFailed(ctx, benchmark, err)
+		return
+	}
+	if !benchmarkPackageDeleteRequired(otherReferences) {
+		task.deleteBenchmark(ctx, benchmark)
+		return
+	}
+
+	pkgObj, err := models.GetLLMBenchmarkPackageManager().FetchById(packageID)
+	if err != nil {
+		cause := errors.Cause(err)
+		if cause == errors.ErrNotFound || cause == sql.ErrNoRows {
+			task.deleteBenchmark(ctx, benchmark)
+			return
+		}
+		task.taskFailed(ctx, benchmark, errors.Wrap(err, "fetch benchmark package"))
+		return
+	}
+	pkg := pkgObj.(*models.SLLMBenchmarkPackage)
+
+	task.SetStage("OnPackageDeleted", nil)
+	if err := pkg.StartDeleteTask(ctx, task.UserCred, nil, task.GetTaskId()); err != nil {
+		task.taskFailed(ctx, benchmark, errors.Wrap(err, "start package delete task"))
+	}
+}
+
+func (task *LLMBenchmarkDeleteTask) OnPackageDeleted(ctx context.Context, benchmark *models.SLLMBenchmark, body jsonutils.JSONObject) {
+	task.deleteBenchmark(ctx, benchmark)
+}
+
+func (task *LLMBenchmarkDeleteTask) OnPackageDeletedFailed(ctx context.Context, benchmark *models.SLLMBenchmark, body jsonutils.JSONObject) {
+	task.taskFailed(ctx, benchmark, errors.Errorf("benchmark package delete failed: %s", body))
+}

pkg/llm/tasks/llm/llm_benchmark_package_delete_task.go

@@ -0,0 +1,139 @@
+package llm
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/httputils"
+
+	commonapis "yunion.io/x/onecloud/pkg/apis"
+	imageapi "yunion.io/x/onecloud/pkg/apis/image"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/llm/models"
+	"yunion.io/x/onecloud/pkg/llm/options"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	imagemodules "yunion.io/x/onecloud/pkg/mcclient/modules/image"
+	"yunion.io/x/onecloud/pkg/util/logclient"
+)
+
+const (
+	benchmarkPackageDeletePollInterval    = 30 * time.Second
+	benchmarkPackageDeletePollMaxAttempts = 60
+)
+
+type LLMBenchmarkPackageDeleteTask struct {
+	taskman.STask
+}
+
+func init() {
+	taskman.RegisterTask(LLMBenchmarkPackageDeleteTask{})
+}
+
+func benchmarkPackageImageNotFound(err error) bool {
+	if err == nil {
+		return false
+	}
+	if httputils.ErrorCode(err) == 404 {
+		return true
+	}
+	return strings.Contains(err.Error(), "ResourceNotFoundError")
+}
+
+func (task *LLMBenchmarkPackageDeleteTask) taskFailed(ctx context.Context, pkg *models.SLLMBenchmarkPackage, err error) {
+	_ = pkg.SetStatus(ctx, task.UserCred, commonapis.STATUS_DELETE_FAILED, err.Error())
+	db.OpsLog.LogEvent(pkg, db.ACT_DELETE_FAIL, err, task.UserCred)
+	logclient.AddActionLogWithStartable(task, pkg, logclient.ACT_DELETE, err, task.UserCred, false)
+	task.SetStageFailed(ctx, jsonutils.NewString(err.Error()))
+}
+
+func (task *LLMBenchmarkPackageDeleteTask) imageID(pkg *models.SLLMBenchmarkPackage) string {
+	if imageID, _ := task.Params.GetString("image_id"); imageID != "" {
+		return imageID
+	}
+	return pkg.ImageId
+}
+
+func (task *LLMBenchmarkPackageDeleteTask) waitImageDeleted(ctx context.Context, imageID string) error {
+	session := auth.GetAdminSession(ctx, options.Options.Region)
+	var lastErr error
+	for i := 0; i < benchmarkPackageDeletePollMaxAttempts; i++ {
+		_, err := imagemodules.Images.Get(session, imageID, nil)
+		if benchmarkPackageImageNotFound(err) {
+			return nil
+		}
+		if err != nil {
+			lastErr = err
+		}
+		time.Sleep(benchmarkPackageDeletePollInterval)
+	}
+	if lastErr != nil {
+		return errors.Wrapf(lastErr, "wait glance image %s deleted", imageID)
+	}
+	return errors.Errorf("wait glance image %s deleted timeout", imageID)
+}
+
+func (task *LLMBenchmarkPackageDeleteTask) deleteImage(ctx context.Context, session *mcclient.ClientSession, imageID string) error {
+	protected := false
+	_, err := imagemodules.Images.Update(session, imageID, jsonutils.Marshal(imageapi.ImageUpdateInput{
+		Protected: &protected,
+	}))
+	if err != nil && !benchmarkPackageImageNotFound(err) {
+		return errors.Wrapf(err, "unprotect glance image %s", imageID)
+	}
+
+	params := jsonutils.NewDict()
+	params.Set("override_pending_delete", jsonutils.JSONTrue)
+	if jsonutils.QueryBoolean(task.Params, "purge", false) {
+		params.Set("purge", jsonutils.JSONTrue)
+	}
+	_, err = imagemodules.Images.DeleteWithParam(session, imageID, params, nil)
+	if err != nil && !benchmarkPackageImageNotFound(err) {
+		return errors.Wrapf(err, "delete glance image %s", imageID)
+	}
+	return task.waitImageDeleted(ctx, imageID)
+}
+
+func (task *LLMBenchmarkPackageDeleteTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
+	pkg := obj.(*models.SLLMBenchmarkPackage)
+	_ = pkg.SetStatus(ctx, task.UserCred, commonapis.STATUS_DELETING, "start delete")
+
+	imageID := task.imageID(pkg)
+	if imageID == "" {
+		task.OnImageDeleted(ctx, pkg, nil)
+		return
+	}
+
+	session := auth.GetAdminSession(ctx, options.Options.Region)
+	_, err := imagemodules.Images.Get(session, imageID, nil)
+	if benchmarkPackageImageNotFound(err) {
+		task.OnImageDeleted(ctx, pkg, nil)
+		return
+	}
+	if err != nil {
+		task.taskFailed(ctx, pkg, errors.Wrapf(err, "get glance image %s", imageID))
+		return
+	}
+
+	task.SetStage("OnImageDeleted", nil)
+	taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {
+		return nil, task.deleteImage(ctx, session, imageID)
+	})
+}
+
+func (task *LLMBenchmarkPackageDeleteTask) OnImageDeletedFailed(ctx context.Context, pkg *models.SLLMBenchmarkPackage, body jsonutils.JSONObject) {
+	task.taskFailed(ctx, pkg, errors.Error(body.String()))
+}
+
+func (task *LLMBenchmarkPackageDeleteTask) OnImageDeleted(ctx context.Context, pkg *models.SLLMBenchmarkPackage, body jsonutils.JSONObject) {
+	if err := pkg.RealDelete(ctx, task.UserCred); err != nil {
+		task.taskFailed(ctx, pkg, err)
+		return
+	}
+	logclient.AddActionLogWithStartable(task, pkg, logclient.ACT_DELETE, nil, task.UserCred, true)
+	task.SetStageComplete(ctx, nil)
+}

pkg/llm/tasks/llm/llm_benchmark_package_import_task.go

@@ -0,0 +1,97 @@
+package llm
+
+import (
+	"context"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+
+	imageapi "yunion.io/x/onecloud/pkg/apis/image"
+	api "yunion.io/x/onecloud/pkg/apis/llm"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/llm/models"
+	"yunion.io/x/onecloud/pkg/llm/options"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	"yunion.io/x/onecloud/pkg/util/logclient"
+)
+
+type LLMBenchmarkPackageImportTask struct {
+	taskman.STask
+}
+
+func init() {
+	taskman.RegisterTask(LLMBenchmarkPackageImportTask{})
+}
+
+func (task *LLMBenchmarkPackageImportTask) taskFailed(ctx context.Context, pkg *models.SLLMBenchmarkPackage, err error, cleanup bool) {
+	_ = pkg.SetStatus(ctx, task.UserCred, imageapi.IMAGE_STATUS_KILLED, err.Error())
+	db.OpsLog.LogEvent(pkg, db.ACT_CREATE, err, task.UserCred)
+	logclient.AddActionLogWithStartable(task, pkg, logclient.ACT_CREATE, err, task.UserCred, false)
+	if cleanup {
+		if cleanupErr := pkg.StartDeleteTask(ctx, task.UserCred, nil, ""); cleanupErr != nil {
+			log.Errorf("cleanup benchmark package %s: %s", pkg.Id, cleanupErr)
+		}
+	}
+	task.SetStageFailed(ctx, jsonutils.NewString(err.Error()))
+}
+
+func (task *LLMBenchmarkPackageImportTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
+	pkg := obj.(*models.SLLMBenchmarkPackage)
+	input := api.LLMBenchmarkPackageImportInput{}
+	if err := task.Params.Unmarshal(&input, "import_input"); err != nil {
+		task.taskFailed(ctx, pkg, err, false)
+		return
+	}
+
+	task.SetStage("OnImportComplete", nil)
+	session := auth.GetAdminSession(ctx, options.Options.Region)
+	tmpDir, err := pkg.DoImport(ctx, task.UserCred, session, input.LLMBenchmarkPackageCreateInput)
+	if tmpDir != "" {
+		defer pkg.CleanupImportTmpDir(ctx, task.UserCred, tmpDir)
+	}
+	if err != nil {
+		task.OnImportCompleteFailed(ctx, pkg, jsonutils.NewString(err.Error()))
+		return
+	}
+	task.OnImportComplete(ctx, pkg, nil)
+}
+
+func (task *LLMBenchmarkPackageImportTask) OnImportComplete(ctx context.Context, pkg *models.SLLMBenchmarkPackage, body jsonutils.JSONObject) {
+	input := api.LLMBenchmarkPackageImportInput{}
+	if err := task.Params.Unmarshal(&input, "import_input"); err != nil {
+		task.taskFailed(ctx, pkg, err, false)
+		return
+	}
+
+	result := jsonutils.NewDict()
+	if input.BenchmarkSpec != nil {
+		benchmarkInput, err := models.PrepareLLMBenchmarkCreateInput(pkg, input.BenchmarkSpec)
+		if err != nil {
+			task.taskFailed(ctx, pkg, err, true)
+			return
+		}
+		benchmark, err := models.GetLLMBenchmarkManager().CreateAndStart(ctx, task.UserCred, pkg.GetOwnerId(), benchmarkInput)
+		if err != nil {
+			task.taskFailed(ctx, pkg, errors.Wrap(err, "create benchmark"), true)
+			return
+		}
+		result.Set("benchmark_id", jsonutils.NewString(benchmark.Id))
+	}
+
+	db.OpsLog.LogEvent(pkg, db.ACT_CREATE, pkg.GetShortDesc(ctx), task.UserCred)
+	logclient.AddActionLogWithStartable(task, pkg, logclient.ACT_CREATE, pkg.GetShortDesc(ctx), task.UserCred, true)
+	task.SetStageComplete(ctx, result)
+}
+
+func (task *LLMBenchmarkPackageImportTask) OnImportCompleteFailed(ctx context.Context, pkg *models.SLLMBenchmarkPackage, body jsonutils.JSONObject) {
+	input := api.LLMBenchmarkPackageImportInput{}
+	_ = task.Params.Unmarshal(&input, "import_input")
+	task.taskFailed(
+		ctx,
+		pkg,
+		errors.Errorf("benchmark package import failed: %s", body),
+		input.BenchmarkSpec != nil,
+	)
+}

pkg/llm/tasks/llm/llm_benchmark_run_task.go

@@ -0,0 +1,687 @@
+package llm
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+
+	commonapi "yunion.io/x/onecloud/pkg/apis"
+	computeapi "yunion.io/x/onecloud/pkg/apis/compute"
+	api "yunion.io/x/onecloud/pkg/apis/llm"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	bench "yunion.io/x/onecloud/pkg/llm/benchmark"
+	"yunion.io/x/onecloud/pkg/llm/models"
+	"yunion.io/x/onecloud/pkg/llm/options"
+	llmutils "yunion.io/x/onecloud/pkg/llm/utils"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	computemod "yunion.io/x/onecloud/pkg/mcclient/modules/compute"
+)
+
+type LLMBenchmarkRunTask struct {
+	taskman.STask
+}
+
+const (
+	preflightRemoteJSON = "/workdir/dataset-preflight.json"
+	preflightRemoteCSV  = "/workdir/dataset-preflight.csv"
+	preflightRemoteLog  = "/workdir/dataset-preflight.log"
+)
+
+const (
+	evaluationDatasetRemote = "/workdir/evaluation-dataset.jsonl"
+	evaluationDatasetLocal  = "evaluation-dataset.jsonl"
+)
+
+func init() {
+	taskman.RegisterTask(LLMBenchmarkRunTask{})
+}
+
+func applyBenchmarkRuntimeTokenizer(spec *bench.GuideLLMSpec, mount *models.LLMBenchmarkTokenizerMount) {
+	if spec == nil || mount == nil {
+		return
+	}
+	spec.Tokenizer = bench.GuideLLMLocalTokenizer(mount.ModelPath)
+}
+
+func (task *LLMBenchmarkRunTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
+	b := obj.(*models.SLLMBenchmark)
+	if stoppedBenchmark(b) {
+		_, _ = b.FinishRun(ctx, nil)
+		task.SetStageComplete(ctx, nil)
+		return
+	}
+	_ = b.SetState(ctx, task.UserCred, api.LLMBenchmarkStateQueued, "")
+	task.SetStage("OnRunComplete", nil)
+	taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {
+		defer cleanupRunner(ctx, task.UserCred, b)
+		err := task.run(ctx, b)
+		persistErr := task.persistBenchmarkArtifacts(ctx, b)
+		if err == nil {
+			err = persistErr
+		}
+		return nil, err
+	})
+}
+
+func (task *LLMBenchmarkRunTask) OnRunComplete(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
+	b := obj.(*models.SLLMBenchmark)
+	if _, err := b.FinishRun(ctx, nil); err != nil {
+		task.SetStageFailed(ctx, jsonutils.NewString(err.Error()))
+		return
+	}
+	task.SetStageComplete(ctx, nil)
+}
+
+func (task *LLMBenchmarkRunTask) OnRunCompleteFailed(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
+	b := obj.(*models.SLLMBenchmark)
+	reason, _ := body.GetString("__reason__")
+	if reason == "" {
+		reason = body.String()
+	}
+	state, err := b.FinishRun(ctx, errors.Error(reason))
+	if err != nil {
+		task.SetStageFailed(ctx, jsonutils.NewString(err.Error()))
+		return
+	}
+	if state == api.LLMBenchmarkStateStopped {
+		task.SetStageComplete(ctx, nil)
+		return
+	}
+	task.SetStageFailed(ctx, jsonutils.NewString(reason))
+}
+
+func (task *LLMBenchmarkRunTask) run(ctx context.Context, b *models.SLLMBenchmark) error {
+	if err := os.MkdirAll(b.WorkDir, 0755); err != nil {
+		return errors.Wrap(err, "MkdirAll")
+	}
+	if err := os.WriteFile(filepath.Join(b.WorkDir, "target_snapshot.json"), []byte(b.TargetSnapshot), 0644); err != nil {
+		return errors.Wrap(err, "write target_snapshot")
+	}
+	llmObj, err := models.GetLLMManager().FetchById(b.LLMId)
+	if err != nil {
+		return errors.Wrap(err, "fetch LLM")
+	}
+	llm := llmObj.(*models.SLLM)
+	targetServer, err := llm.GetServer(ctx)
+	if err != nil {
+		return errors.Wrap(err, "GetServer")
+	}
+	imageObj, err := models.GetLLMImageManager().FetchById(b.LLMImageId)
+	if err != nil {
+		return errors.Wrap(err, "FetchById benchmark image")
+	}
+	image := imageObj.(*models.SLLMImage)
+
+	var spec bench.GuideLLMSpec
+	if err := json.Unmarshal([]byte(b.GuideLLMSpec), &spec); err != nil {
+		return errors.Wrap(err, "unmarshal GuideLLMSpec")
+	}
+	var tokenizerMount *models.LLMBenchmarkTokenizerMount
+	if b.DatasetName == api.LLMBenchmarkDatasetSyntheticText {
+		tokenizerMount, err = b.ResolveTokenizerMount()
+		if err != nil {
+			return errors.Wrap(err, "resolve offline benchmark tokenizer")
+		}
+		applyBenchmarkRuntimeTokenizer(&spec, tokenizerMount)
+	}
+	runtimeSpec, err := json.Marshal(spec)
+	if err != nil {
+		return errors.Wrap(err, "marshal runtime GuideLLMSpec")
+	}
+	if err := os.WriteFile(filepath.Join(b.WorkDir, "spec.json"), runtimeSpec, 0644); err != nil {
+		return errors.Wrap(err, "write spec")
+	}
+	envs, err := bench.GuideLLMEnvs(spec)
+	if err != nil {
+		return errors.Wrap(err, "GuideLLMEnvs")
+	}
+	podInput, err := b.RunnerPodInput(image.ToContainerImage(), targetServer, tokenizerMount)
+	if err != nil {
+		return err
+	}
+	podInput.Pod.Containers[0].Envs = append(podInput.Pod.Containers[0].Envs, envMapToKVs(envs)...)
+	if stoppedBenchmark(b) {
+		return errors.Error("benchmark stopped")
+	}
+
+	s := auth.GetSession(ctx, task.UserCred, options.Options.Region)
+	resp, err := computemod.Servers.Create(s, jsonutils.Marshal(podInput))
+	if err != nil {
+		return errors.Wrap(err, "create runner pod")
+	}
+	runnerServerId, err := resp.GetString("id")
+	if err != nil {
+		return errors.Wrap(err, "runner server id")
+	}
+	_ = b.SetRunner(ctx, runnerServerId, "")
+	if stoppedBenchmark(b) {
+		return errors.Error("benchmark stopped")
+	}
+
+	server, err := llmutils.WaitServerStatus(ctx, runnerServerId, []string{computeapi.VM_RUNNING}, 600)
+	if err != nil {
+		return errors.Wrap(err, "wait runner server")
+	}
+	if len(server.Containers) == 0 {
+		return errors.Error("runner pod has no containers")
+	}
+	containerId := server.Containers[0].Id
+	_ = b.SetRunner(ctx, runnerServerId, containerId)
+	if stoppedBenchmark(b) {
+		return errors.Error("benchmark stopped")
+	}
+	if b.DatasetName == api.LLMBenchmarkDatasetPackage {
+		if err := task.runDatasetPreflight(ctx, s, containerId, b, spec); err != nil {
+			return err
+		}
+	}
+	if stoppedBenchmark(b) {
+		return errors.Error("benchmark stopped")
+	}
+	if err := b.SetState(ctx, task.UserCred, api.LLMBenchmarkStateRunning, ""); err != nil {
+		return err
+	}
+
+	phaseErr := executeGuideLLMPhase(
+		ctx,
+		s,
+		containerId,
+		b,
+		bench.GuideLLMRunCommand(),
+		"/workdir/benchmarks.json",
+		"/workdir/benchmarks.csv",
+	)
+	if phaseErr != nil {
+		_ = copyBenchmarkArtifacts(s, containerId, b)
+		return phaseErr
+	}
+	if err := copyBenchmarkArtifacts(s, containerId, b); err != nil {
+		return err
+	}
+
+	metrics, err := bench.ParseMetricsCSV(b.ResultCsv)
+	if err != nil {
+		_ = b.SetState(ctx, task.UserCred, api.LLMBenchmarkStateRunning, "parse metrics: "+err.Error())
+		if b.DatasetName == api.LLMBenchmarkDatasetPackage {
+			return task.saveDatasetEvaluationFailure(ctx, b, "", 0, errors.Wrap(err, "parse formal metrics"))
+		}
+		return nil
+	}
+	if err := b.UpdateMetrics(ctx, task.UserCred, metrics); err != nil {
+		return err
+	}
+	if b.DatasetName == api.LLMBenchmarkDatasetPackage {
+		return task.runDatasetEvaluation(ctx, s, containerId, b, metrics.RequestTotal)
+	}
+	return nil
+}
+
+func evaluationDatasetExportCommand(datasetPath string, requestTotal int) []string {
+	command := fmt.Sprintf(
+		"head -n %d %s > %s",
+		requestTotal,
+		shellPath(datasetPath),
+		evaluationDatasetRemote,
+	)
+	return []string{"sh", "-lc", command}
+}
+
+func datasetEvaluationFailure(
+	workDir, answerColumn string,
+	requestTotal int,
+	cause error,
+) (bench.EvaluationResult, error) {
+	ret := bench.EvaluationResult{
+		Summary: api.LLMBenchmarkDatasetEvaluation{
+			State:        api.LLMBenchmarkEvaluationStateError,
+			AnswerColumn: answerColumn,
+			RequestTotal: requestTotal,
+			Message:      cause.Error(),
+		},
+		LogPath: filepath.Join(workDir, "evaluation.log"),
+	}
+	err := os.WriteFile(
+		ret.LogPath,
+		[]byte("evaluation error: "+cause.Error()+"\n"),
+		0644,
+	)
+	return ret, err
+}
+
+func appendArtifactStorageFallback(logPath string, cause error) {
+	if logPath == "" || cause == nil {
+		return
+	}
+	file, err := os.OpenFile(logPath, os.O_APPEND|os.O_WRONLY, 0644)
+	if err != nil {
+		return
+	}
+	defer file.Close()
+	_, _ = fmt.Fprintf(file, "artifact storage fallback: %s\n", cause)
+}
+
+func exportEvaluationDataset(
+	s *mcclient.ClientSession,
+	containerID, datasetPath string,
+	requestTotal int,
+) error {
+	if requestTotal <= 0 {
+		return errors.Error("formal benchmark executed zero requests")
+	}
+	input := &computeapi.ContainerExecSyncInput{
+		Command: evaluationDatasetExportCommand(datasetPath, requestTotal),
+		Timeout: 60,
+	}
+	response, err := computemod.Containers.PerformAction(
+		s,
+		containerID,
+		"exec-sync",
+		jsonutils.Marshal(input),
+	)
+	if err != nil {
+		return errors.Wrap(err, "export evaluation dataset")
+	}
+	if code, _ := response.Int("exit_code"); code != 0 {
+		stderr, _ := response.GetString("stderr")
+		return fmt.Errorf("export evaluation dataset exited %d: %s", code, stderr)
+	}
+	return nil
+}
+
+func (task *LLMBenchmarkRunTask) runDatasetEvaluation(
+	ctx context.Context,
+	s *mcclient.ClientSession,
+	containerID string,
+	b *models.SLLMBenchmark,
+	requestTotal int,
+) error {
+	evaluating := api.LLMBenchmarkDatasetEvaluation{
+		State:        api.LLMBenchmarkEvaluationStateEvaluating,
+		RequestTotal: requestTotal,
+	}
+	if err := b.UpdateDatasetEvaluation(ctx, task.UserCred, evaluating, "", "", ""); err != nil {
+		return err
+	}
+
+	pkgObj, err := models.GetLLMBenchmarkPackageManager().FetchById(b.BenchmarkPackageId)
+	if err != nil {
+		return task.saveDatasetEvaluationFailure(ctx, b, "", requestTotal, errors.Wrap(err, "fetch benchmark package"))
+	}
+	pkg := pkgObj.(*models.SLLMBenchmarkPackage)
+	if err := exportEvaluationDataset(s, containerID, b.DatasetPath, requestTotal); err != nil {
+		return task.saveDatasetEvaluationFailure(ctx, b, pkg.AnswerColumn, requestTotal, err)
+	}
+	localDataset := filepath.Join(b.WorkDir, evaluationDatasetLocal)
+	copied, err := copyArtifacts(s, containerID, map[string]string{
+		evaluationDatasetRemote: localDataset,
+	})
+	if err != nil || copied[evaluationDatasetRemote] == "" {
+		if err == nil {
+			err = errors.Error("evaluation dataset artifact not copied")
+		}
+		return task.saveDatasetEvaluationFailure(ctx, b, pkg.AnswerColumn, requestTotal, err)
+	}
+	defer os.Remove(localDataset)
+
+	result, evaluationErr := bench.EvaluateDataset(ctx, bench.EvaluationInput{
+		DatasetPath:   localDataset,
+		BenchmarkPath: b.ResultJson,
+		AnswerColumn:  pkg.AnswerColumn,
+		OutputDir:     b.WorkDir,
+	})
+	if evaluationErr != nil && result.Summary.State == "" {
+		return task.saveDatasetEvaluationFailure(ctx, b, pkg.AnswerColumn, requestTotal, evaluationErr)
+	}
+	if result.Summary.State != api.LLMBenchmarkEvaluationStateCompleted {
+		result.Summary.RequestTotal = requestTotal
+	}
+	if err := b.UpdateDatasetEvaluation(
+		ctx,
+		task.UserCred,
+		result.Summary,
+		result.ResultJSON,
+		result.ResultCSV,
+		result.LogPath,
+	); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (task *LLMBenchmarkRunTask) saveDatasetEvaluationFailure(
+	ctx context.Context,
+	b *models.SLLMBenchmark,
+	answerColumn string,
+	requestTotal int,
+	cause error,
+) error {
+	result, writeErr := datasetEvaluationFailure(b.WorkDir, answerColumn, requestTotal, cause)
+	if writeErr != nil {
+		result.Summary.Message += ": write evaluation log: " + writeErr.Error()
+		result.LogPath = ""
+	}
+	return b.UpdateDatasetEvaluation(
+		ctx,
+		task.UserCred,
+		result.Summary,
+		"",
+		"",
+		result.LogPath,
+	)
+}
+
+func (task *LLMBenchmarkRunTask) persistBenchmarkArtifacts(
+	ctx context.Context,
+	b *models.SLLMBenchmark,
+) error {
+	local := b.ArtifactLocations()
+	store := bench.DefaultArtifactStore()
+	stored, storage, storageErr := store.Persist(ctx, b.ProjectId, b.Id, local)
+	message := ""
+	if storageErr != nil {
+		message = storageErr.Error()
+		appendArtifactStorageFallback(local["evaluation-log"], storageErr)
+	}
+	if err := b.UpdateArtifactLocations(ctx, task.UserCred, stored, storage, message); err != nil {
+		if storage == api.LLMBenchmarkArtifactStorageMinio {
+			_ = store.DeleteBenchmark(ctx, b.ProjectId, b.Id)
+		}
+		return err
+	}
+	if storage == api.LLMBenchmarkArtifactStorageMinio {
+		if err := store.RemoveLocal(local); err != nil {
+			log.Warningf("remove uploaded benchmark %s local artifacts: %s", b.Id, err)
+		}
+	}
+	return nil
+}
+
+func evaluateDatasetPreflight(metrics *bench.LLMBenchmarkMetrics, phaseErr error) (api.LLMBenchmarkDatasetPreflight, error) {
+	ret := api.LLMBenchmarkDatasetPreflight{
+		State:           api.LLMBenchmarkStateCompleted,
+		ExpectedSamples: bench.DatasetPreflightSamples,
+	}
+	if metrics == nil {
+		ret.State = api.LLMBenchmarkStateError
+		if phaseErr == nil {
+			phaseErr = errors.Error("dataset preflight metrics not ready")
+		}
+		ret.Message = phaseErr.Error()
+		return ret, phaseErr
+	}
+	ret.ActualSamples = metrics.RequestTotal
+	ret.Successful = metrics.RequestSuccessful
+	ret.Errored = metrics.RequestErrored
+	if metrics.ErrorRate != nil {
+		ret.ErrorRate = *metrics.ErrorRate
+	} else if metrics.RequestTotal > 0 {
+		ret.ErrorRate = float64(metrics.RequestErrored) / float64(metrics.RequestTotal)
+	}
+	if metrics.RequestLatencyMeanSec != nil {
+		ret.LatencyMeanSeconds = *metrics.RequestLatencyMeanSec
+	}
+	if metrics.RequestTotal == 0 {
+		ret.State = api.LLMBenchmarkStateError
+		ret.Message = "dataset preflight executed zero requests"
+		return ret, errors.Error(ret.Message)
+	}
+	if metrics.RequestSuccessful == 0 {
+		ret.State = api.LLMBenchmarkStateError
+		ret.Message = fmt.Sprintf("dataset preflight failed all %d requests", metrics.RequestTotal)
+		if phaseErr != nil {
+			ret.Message += ": " + phaseErr.Error()
+		}
+		return ret, errors.Error(ret.Message)
+	}
+	if metrics.RequestErrored > 0 || phaseErr != nil {
+		ret.Message = fmt.Sprintf("dataset preflight completed with %d successful and %d errored requests", metrics.RequestSuccessful, metrics.RequestErrored)
+		if phaseErr != nil {
+			ret.Message += ": " + phaseErr.Error()
+		}
+	}
+	return ret, nil
+}
+
+func (task *LLMBenchmarkRunTask) runDatasetPreflight(
+	ctx context.Context,
+	s *mcclient.ClientSession,
+	containerId string,
+	b *models.SLLMBenchmark,
+	formal bench.GuideLLMSpec,
+) error {
+	preflight, err := bench.BuildGuideLLMPreflightSpec(formal, b.MaxDurationSeconds)
+	if err != nil {
+		return err
+	}
+	command, err := bench.GuideLLMPreflightRunCommand(preflight)
+	if err != nil {
+		return err
+	}
+	if err := b.SetState(ctx, task.UserCred, api.LLMBenchmarkStateValidating, ""); err != nil {
+		return err
+	}
+	validating := api.LLMBenchmarkDatasetPreflight{
+		State:           api.LLMBenchmarkStateValidating,
+		ExpectedSamples: bench.DatasetPreflightSamples,
+	}
+	if err := b.UpdateDatasetPreflight(ctx, task.UserCred, validating, "", ""); err != nil {
+		return errors.Wrap(err, "start dataset preflight")
+	}
+
+	phaseErr := executeGuideLLMPhase(ctx, s, containerId, b, command, preflightRemoteJSON, preflightRemoteCSV)
+	paths := map[string]string{
+		preflightRemoteJSON: filepath.Join(b.WorkDir, "dataset-preflight.json"),
+		preflightRemoteCSV:  filepath.Join(b.WorkDir, "dataset-preflight.csv"),
+		preflightRemoteLog:  filepath.Join(b.WorkDir, "dataset-preflight.log"),
+	}
+	copied, copyErr := copyArtifacts(s, containerId, paths)
+	if phaseErr == nil {
+		phaseErr = copyErr
+	}
+
+	if stoppedBenchmark(b) {
+		summary := api.LLMBenchmarkDatasetPreflight{
+			State:           api.LLMBenchmarkStateStopped,
+			ExpectedSamples: bench.DatasetPreflightSamples,
+			Message:         "benchmark stopped during dataset preflight",
+		}
+		_ = b.UpdateDatasetPreflight(ctx, task.UserCred, summary, copied[preflightRemoteJSON], copied[preflightRemoteLog])
+		return errors.Error(summary.Message)
+	}
+
+	var metrics *bench.LLMBenchmarkMetrics
+	if csvPath := copied[preflightRemoteCSV]; csvPath != "" {
+		metrics, err = bench.ParseMetricsCSV(csvPath)
+		if phaseErr == nil {
+			phaseErr = err
+		}
+	}
+	if metrics == nil || metrics.RequestSuccessful == 0 {
+		phaseErr = datasetPreflightPhaseError(phaseErr, copied[preflightRemoteLog])
+	}
+	summary, decisionErr := evaluateDatasetPreflight(metrics, phaseErr)
+	if err := b.UpdateDatasetPreflight(ctx, task.UserCred, summary, copied[preflightRemoteJSON], copied[preflightRemoteLog]); err != nil {
+		return errors.Wrap(err, "save dataset preflight")
+	}
+	return decisionErr
+}
+
+func datasetPreflightPhaseError(phaseErr error, logPath string) error {
+	f, err := os.Open(logPath)
+	if err != nil {
+		return phaseErr
+	}
+	defer f.Close()
+
+	scanner := bufio.NewScanner(f)
+	scanner.Buffer(make([]byte, 64*1024), 1024*1024)
+	last := ""
+	for scanner.Scan() {
+		if line := strings.TrimSpace(scanner.Text()); line != "" {
+			last = line
+		}
+	}
+	if last == "" {
+		return phaseErr
+	}
+	if phaseErr == nil {
+		return errors.Error(last)
+	}
+	if strings.Contains(phaseErr.Error(), last) {
+		return phaseErr
+	}
+	return errors.Wrap(phaseErr, last)
+}
+
+func executeGuideLLMPhase(
+	ctx context.Context,
+	s *mcclient.ClientSession,
+	containerId string,
+	b *models.SLLMBenchmark,
+	command []string,
+	jsonPath, csvPath string,
+) error {
+	execInput := &computeapi.ContainerExecSyncInput{
+		Command: command,
+		Timeout: int64(b.MaxDurationSeconds + 120),
+	}
+	execResp, err := computemod.Containers.PerformAction(s, containerId, "exec-sync", jsonutils.Marshal(execInput))
+	if err != nil {
+		if waitBenchmarkResults(ctx, s, containerId, b, jsonPath, csvPath, b.MaxDurationSeconds+30) != nil {
+			return errors.Wrap(err, "guidellm exec")
+		}
+		return nil
+	}
+	if code, _ := execResp.Int("exit_code"); code != 0 {
+		stderr, _ := execResp.GetString("stderr")
+		return fmt.Errorf("guidellm exited %d: %s", code, stderr)
+	}
+	return nil
+}
+
+func waitBenchmarkResults(ctx context.Context, s *mcclient.ClientSession, containerId string, b *models.SLLMBenchmark, jsonPath, csvPath string, seconds int) error {
+	if seconds <= 0 {
+		seconds = 30
+	}
+	deadline := time.Now().Add(time.Duration(seconds) * time.Second)
+	for time.Now().Before(deadline) {
+		if stoppedBenchmark(b) {
+			return errors.Error("benchmark stopped")
+		}
+		if benchmarkResultsReady(s, containerId, jsonPath, csvPath) {
+			return nil
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(2 * time.Second):
+		}
+	}
+	return errors.Error("benchmark result artifacts not ready")
+}
+
+func benchmarkResultsReady(s *mcclient.ClientSession, containerId, jsonPath, csvPath string) bool {
+	command := fmt.Sprintf("test -s %s -a -s %s", shellPath(jsonPath), shellPath(csvPath))
+	input := &computeapi.ContainerExecSyncInput{
+		Command: []string{"sh", "-lc", command},
+		Timeout: 10,
+	}
+	resp, err := computemod.Containers.PerformAction(s, containerId, "exec-sync", jsonutils.Marshal(input))
+	if err != nil {
+		return false
+	}
+	code, _ := resp.Int("exit_code")
+	return code == 0
+}
+
+func shellPath(path string) string {
+	return "'" + strings.ReplaceAll(path, "'", "'\"'\"'") + "'"
+}
+
+func copyBenchmarkArtifacts(s *mcclient.ClientSession, containerId string, b *models.SLLMBenchmark) error {
+	paths := map[string]string{
+		"/workdir/guidellm.log":    filepath.Join(b.WorkDir, "guidellm.log"),
+		"/workdir/benchmarks.json": filepath.Join(b.WorkDir, "benchmarks.json"),
+		"/workdir/benchmarks.csv":  filepath.Join(b.WorkDir, "benchmarks.csv"),
+	}
+	copied, firstErr := copyArtifacts(s, containerId, paths)
+	_, err := db.Update(b, func() error {
+		applyBenchmarkArtifactPaths(b, copied)
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+	return firstErr
+}
+
+func applyBenchmarkArtifactPaths(b *models.SLLMBenchmark, copied map[string]string) {
+	b.LogPath = copied["/workdir/guidellm.log"]
+	b.ResultJson = copied["/workdir/benchmarks.json"]
+	b.ResultCsv = copied["/workdir/benchmarks.csv"]
+}
+
+func copyArtifacts(s *mcclient.ClientSession, containerId string, paths map[string]string) (map[string]string, error) {
+	copied := map[string]string{}
+	var firstErr error
+	for remote, local := range paths {
+		out, err := os.Create(local)
+		if err != nil {
+			if firstErr == nil {
+				firstErr = errors.Wrapf(err, "create %s", local)
+			}
+			continue
+		}
+		err = computemod.Containers.CopyFrom(s, containerId, remote, out)
+		closeErr := out.Close()
+		if err != nil {
+			if firstErr == nil {
+				firstErr = errors.Wrapf(err, "copy %s", remote)
+			}
+			continue
+		}
+		if closeErr != nil {
+			if firstErr == nil {
+				firstErr = errors.Wrap(closeErr, "close artifact")
+			}
+			continue
+		}
+		copied[remote] = local
+	}
+	return copied, firstErr
+}
+
+func cleanupRunner(ctx context.Context, userCred mcclient.TokenCredential, b *models.SLLMBenchmark) {
+	if b.RunnerServerId == "" {
+		return
+	}
+	_ = b.DeleteRunnerServer(ctx, userCred)
+}
+
+func stoppedBenchmark(b *models.SLLMBenchmark) bool {
+	obj, err := models.GetLLMBenchmarkManager().FetchById(b.Id)
+	if err != nil {
+		return b.StopRequested
+	}
+	cur := obj.(*models.SLLMBenchmark)
+	return cur.StopRequested || cur.State == api.LLMBenchmarkStateStopped
+}
+
+func envMapToKVs(envs map[string]string) []*commonapi.ContainerKeyValue {
+	ret := make([]*commonapi.ContainerKeyValue, 0, len(envs))
+	for k, v := range envs {
+		ret = append(ret, &commonapi.ContainerKeyValue{Key: k, Value: v})
+	}
+	return ret
+}

pkg/mcclient/modules/llm/mod_llm_benchmark.go

@@ -0,0 +1,60 @@
+package llm
+
+import (
+	"fmt"
+	"io"
+	"net/url"
+	"strconv"
+
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+type LLMBenchmarkManager struct {
+	modulebase.ResourceManager
+}
+
+var (
+	LLMBenchmarks LLMBenchmarkManager
+)
+
+func (m *LLMBenchmarkManager) Artifact(s *mcclient.ClientSession, id string, kind string) (io.ReadCloser, int64, error) {
+	path := fmt.Sprintf("/%s/%s/artifacts/%s", m.URLPath(), url.PathEscape(id), url.PathEscape(kind))
+	resp, err := modulebase.RawRequest(m.ResourceManager, s, "GET", path, nil, nil)
+	if err == nil && resp.StatusCode >= 200 && resp.StatusCode < 300 {
+		sizeBytes, err := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64)
+		if err != nil {
+			log.Errorf("download benchmark artifact unknown size")
+			sizeBytes = -1
+		}
+		return resp.Body, sizeBytes, nil
+	}
+	_, _, err = s.ParseJSONResponse("", resp, err)
+	return nil, -1, err
+}
+
+func init() {
+	LLMBenchmarks = LLMBenchmarkManager{
+		ResourceManager: modules.NewLLMManager("llm_benchmark", "llm_benchmarks",
+			[]string{
+				"ID",
+				"Name",
+				"LLMDeploymentId",
+				"LLMDeployment",
+				"LLMId",
+				"State",
+				"RequestRate",
+				"TotalRequests",
+				"RequestTotal",
+				"RequestSuccessful",
+				"RequestErrored",
+				"ErrorRate",
+			},
+			[]string{},
+		),
+	}
+	modules.Register(&LLMBenchmarks)
+}

pkg/mcclient/modules/llm/mod_llm_benchmark_package.go

@@ -0,0 +1,35 @@
+package llm
+
+import (
+	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+type LLMBenchmarkPackageManager struct {
+	modulebase.ResourceManager
+}
+
+var (
+	LLMBenchmarkPackages LLMBenchmarkPackageManager
+)
+
+func init() {
+	LLMBenchmarkPackages = LLMBenchmarkPackageManager{
+		ResourceManager: modules.NewLLMManager("llm_benchmark_package", "llm_benchmark_packages",
+			[]string{
+				"ID",
+				"Name",
+				"Status",
+				"Source",
+				"RepoId",
+				"Revision",
+				"FilePath",
+				"Format",
+				"ImageId",
+				"DatasetPath",
+			},
+			[]string{},
+		),
+	}
+	modules.Register(&LLMBenchmarkPackages)
+}

pkg/mcclient/options/llm/image.go

@@ -18,7 +18,7 @@ func (o *LLMImageShowOptions) Params() (jsonutils.JSONObject, error) {
 type LLMImageListOptions struct {
 	options.BaseListOptions
 
-	LLMType string `json:"llm_type" choices:"ollama|dify|comfyui|vllm|sglang|hermes-agent|llm-router|desktop" help:"filter by llm type"`
+	LLMType string `json:"llm_type" choices:"ollama|dify|comfyui|vllm|sglang|hermes-agent|llm-router|desktop|benchmark" help:"filter by llm type"`
 }
 
 func (o *LLMImageListOptions) Params() (jsonutils.JSONObject, error) {
@@ -30,7 +30,7 @@ type LLMImageCreateOptions struct {
 	IMAGE_NAME   string `json:"image_name"`
 	IMAGE_LABEL  string `json:"image_label"`
 	CredentialId string `json:"credential_id"`
-	LLM_TYPE     string `json:"llm_type" choices:"ollama|dify|comfyui|vllm|sglang|hermes-agent|llm-router|desktop" help:"llm type: ollama, comfyui, vllm, sglang, hermes-agent, llm-router, desktop or dify"`
+	LLM_TYPE     string `json:"llm_type" choices:"ollama|dify|comfyui|vllm|sglang|hermes-agent|llm-router|desktop|benchmark" help:"llm type: ollama, comfyui, vllm, sglang, hermes-agent, llm-router, desktop, benchmark or dify"`
 }
 
 func (o *LLMImageCreateOptions) Params() (jsonutils.JSONObject, error) {
@@ -44,7 +44,7 @@ type LLMImageUpdateOptions struct {
 	ImageName    string `json:"image_name"`
 	ImageLabel   string `json:"image_label"`
 	CredentialId string `json:"credential_id"`
-	LlmType      string `json:"llm_type" choices:"ollama|dify|vllm|sglang|comfyui|hermes-agent|llm-router|desktop" help:"llm type: ollama, comfyui, vllm, sglang, hermes-agent, llm-router, desktop or dify"`
+	LlmType      string `json:"llm_type" choices:"ollama|dify|vllm|sglang|comfyui|hermes-agent|llm-router|desktop|benchmark" help:"llm type: ollama, comfyui, vllm, sglang, hermes-agent, llm-router, desktop, benchmark or dify"`
 	AppName      string `json:"app_name" help:"desktop application identifier (LinuxServer image id), e.g. firefox, chromium, steam, webtop-ubuntu-xfce"`
 }
 

pkg/mcclient/options/llm/llm_benchmark.go

@@ -0,0 +1,134 @@
+package llm
+
+import (
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/apis"
+	"yunion.io/x/onecloud/pkg/mcclient/options"
+)
+
+type LLMBenchmarkShowOptions struct {
+	options.BaseShowOptions
+}
+
+func (o *LLMBenchmarkShowOptions) Params() (jsonutils.JSONObject, error) {
+	return options.ListStructToParams(o)
+}
+
+type LLMBenchmarkListOptions struct {
+	options.BaseListOptions
+
+	LLMId           string `json:"llm_id" token:"llm" help:"filter by LLM id"`
+	LLMDeploymentId string `json:"llm_deployment_id" token:"llm-deployment" help:"filter by LLM deployment id or name"`
+	State           string `json:"state" choices:"pending|queued|running|completed|stopped|error" help:"filter by state"`
+}
+
+func (o *LLMBenchmarkListOptions) Params() (jsonutils.JSONObject, error) {
+	return options.ListStructToParams(o)
+}
+
+type LLMBenchmarkCreateOptions struct {
+	apis.VirtualResourceCreateInput
+
+	LLMId            string `json:"llm_id" token:"llm" help:"target LLM id or name"`
+	LLMDeploymentId  string `json:"llm_deployment_id" token:"llm-deployment" help:"target LLM deployment id or name"`
+	BenchmarkImage   string `json:"benchmark_image" help:"benchmark llm_image id or name"`
+	BenchmarkPackage string `json:"benchmark_package" help:"benchmark package id or name"`
+
+	RequestFormat string `json:"request_format" default:"/v1/chat/completions"`
+	Model         string `json:"model"`
+	Profile       string `json:"profile" choices:"constant"`
+	RequestRate   int    `json:"request_rate"`
+
+	TotalRequests      int `json:"total_requests"`
+	MaxDurationSeconds int `json:"max_duration_seconds"`
+	MaxErrors          int `json:"max_errors"`
+
+	DatasetName         string `json:"dataset_name" choices:"synthetic_text|benchmark_package"`
+	DatasetInputTokens  int    `json:"dataset_input_tokens"`
+	DatasetOutputTokens int    `json:"dataset_output_tokens"`
+	DatasetPath         string `json:"dataset_path"`
+}
+
+func (o *LLMBenchmarkCreateOptions) Params() (jsonutils.JSONObject, error) {
+	return jsonutils.Marshal(o), nil
+}
+
+type LLMBenchmarkUpdateOptions struct {
+	options.BaseIdOptions
+
+	Name                string  `json:"name,omitempty"`
+	Description         *string `json:"description,omitempty"`
+	Model               *string `json:"model,omitempty"`
+	RequestRate         *int    `json:"request_rate,omitempty"`
+	TotalRequests       *int    `json:"total_requests,omitempty"`
+	MaxDurationSeconds  *int    `json:"max_duration_seconds,omitempty"`
+	MaxErrors           *int    `json:"max_errors,omitempty"`
+	DatasetInputTokens  *int    `json:"dataset_input_tokens,omitempty"`
+	DatasetOutputTokens *int    `json:"dataset_output_tokens,omitempty"`
+}
+
+func (o *LLMBenchmarkUpdateOptions) GetId() string {
+	return o.ID
+}
+
+func (o *LLMBenchmarkUpdateOptions) Params() (jsonutils.JSONObject, error) {
+	return options.StructToParams(o)
+}
+
+type LLMBenchmarkCopyOptions struct {
+	options.BaseIdOptions
+
+	NAME                string  `json:"name"`
+	LLMDeploymentId     string  `json:"llm_deployment_id"`
+	Description         *string `json:"description,omitempty"`
+	Model               *string `json:"model,omitempty"`
+	RequestRate         *int    `json:"request_rate,omitempty"`
+	TotalRequests       *int    `json:"total_requests,omitempty"`
+	MaxDurationSeconds  *int    `json:"max_duration_seconds,omitempty"`
+	MaxErrors           *int    `json:"max_errors,omitempty"`
+	DatasetInputTokens  *int    `json:"dataset_input_tokens,omitempty"`
+	DatasetOutputTokens *int    `json:"dataset_output_tokens,omitempty"`
+}
+
+func (o *LLMBenchmarkCopyOptions) GetId() string {
+	return o.ID
+}
+
+func (o *LLMBenchmarkCopyOptions) Params() (jsonutils.JSONObject, error) {
+	return options.StructToParams(o)
+}
+
+type LLMBenchmarkRetestOptions struct {
+	options.BaseIdOptions
+}
+
+func (o *LLMBenchmarkRetestOptions) GetId() string {
+	return o.ID
+}
+
+func (o *LLMBenchmarkRetestOptions) Params() (jsonutils.JSONObject, error) {
+	return jsonutils.NewDict(), nil
+}
+
+type LLMBenchmarkDeleteOptions struct {
+	options.BaseIdOptions
+}
+
+func (o *LLMBenchmarkDeleteOptions) GetId() string {
+	return o.ID
+}
+
+type LLMBenchmarkStopOptions struct {
+	options.BaseIdOptions
+}
+
+func (o *LLMBenchmarkStopOptions) GetId() string {
+	return o.ID
+}
+
+type LLMBenchmarkArtifactOptions struct {
+	ID     string `help:"ID or name of benchmark"`
+	Type   string `help:"artifact type" choices:"log|json|csv"`
+	Output string `short-token:"o" help:"destination file, if omitted, output to stdout"`
+}

pkg/mcclient/options/llm/llm_benchmark_package.go

@@ -0,0 +1,73 @@
+package llm
+
+import (
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/apis"
+	api "yunion.io/x/onecloud/pkg/apis/llm"
+	"yunion.io/x/onecloud/pkg/mcclient/options"
+)
+
+type LLMBenchmarkPackageShowOptions struct {
+	options.BaseShowOptions
+}
+
+func (o *LLMBenchmarkPackageShowOptions) Params() (jsonutils.JSONObject, error) {
+	return options.StructToParams(o)
+}
+
+type LLMBenchmarkPackageListOptions struct {
+	options.BaseListOptions
+
+	Source string `json:"source" choices:"huggingface"`
+	RepoId string `json:"repo_id"`
+	Format string `json:"format" choices:"guidellm_jsonl"`
+}
+
+func (o *LLMBenchmarkPackageListOptions) Params() (jsonutils.JSONObject, error) {
+	return options.ListStructToParams(o)
+}
+
+type LLMBenchmarkPackageCreateOptions struct {
+	apis.SharableVirtualResourceCreateInput
+
+	Source   string `json:"source" choices:"huggingface"`
+	RepoId   string `json:"repo_id"`
+	Revision string `json:"revision"`
+	FilePath string `json:"file_path"`
+	Format   string `json:"format" choices:"guidellm_jsonl"`
+	ImageId  string `json:"image_id"`
+}
+
+func (o *LLMBenchmarkPackageCreateOptions) Params() (jsonutils.JSONObject, error) {
+	return jsonutils.Marshal(o), nil
+}
+
+type LLMBenchmarkPackageImportOptions struct {
+	NAME      string `json:"name" help:"benchmark package name"`
+	REPO_ID   string `json:"repo_id" help:"HuggingFace dataset repo id, e.g. org/dataset"`
+	FILE_PATH string `json:"file_path" help:"JSONL file path in the dataset repo"`
+	Revision  string `json:"revision" help:"HuggingFace revision, default main"`
+}
+
+func (o *LLMBenchmarkPackageImportOptions) Params() (jsonutils.JSONObject, error) {
+	input := api.LLMBenchmarkPackageImportInput{
+		LLMBenchmarkPackageCreateInput: api.LLMBenchmarkPackageCreateInput{
+			Source:   api.LLMBenchmarkPackageSourceHuggingFace,
+			RepoId:   o.REPO_ID,
+			Revision: o.Revision,
+			FilePath: o.FILE_PATH,
+			Format:   api.LLMBenchmarkPackageFormatGuideLLMJSONL,
+		},
+	}
+	input.Name = o.NAME
+	return jsonutils.Marshal(input), nil
+}
+
+type LLMBenchmarkPackageDeleteOptions struct {
+	options.BaseIdOptions
+}
+
+func (o *LLMBenchmarkPackageDeleteOptions) GetId() string {
+	return o.ID
+}