From 20620bb5a12d5715a1d084e9aa5b18c97dd82512 Mon Sep 17 00:00:00 2001 From: cwz_eikoh Date: Wed, 15 Jul 2026 15:00:37 +0800 Subject: [PATCH] feat(llm): support llm-bench in llm (#25103) --- cmd/climc/shell/llm/llm_benchmark.go | 55 + cmd/climc/shell/llm/llm_benchmark_package.go | 16 + .../handler/llm_benchmark_artifact.go | 75 + pkg/apigateway/handler/misc.go | 4 + pkg/apis/llm/image.go | 2 + pkg/apis/llm/llm_benchmark.go | 166 ++ pkg/apis/llm/llm_benchmark_package.go | 42 + pkg/llm/benchmark/artifact_store.go | 286 ++++ pkg/llm/benchmark/doc.go | 1 + pkg/llm/benchmark/evaluation.go | 772 +++++++++ pkg/llm/benchmark/guidellm.go | 404 +++++ pkg/llm/models/llm_benchmark.go | 1418 +++++++++++++++++ pkg/llm/models/llm_benchmark_package.go | 520 ++++++ pkg/llm/models/llm_benchmark_stop_state.go | 13 + pkg/llm/options/option.go | 18 + pkg/llm/service/benchmark_handler.go | 214 +++ pkg/llm/service/handler.go | 3 + pkg/llm/service/service.go | 9 + .../tasks/llm/llm_benchmark_delete_task.go | 96 ++ .../llm/llm_benchmark_package_delete_task.go | 139 ++ .../llm/llm_benchmark_package_import_task.go | 97 ++ pkg/llm/tasks/llm/llm_benchmark_run_task.go | 687 ++++++++ pkg/mcclient/modules/llm/mod_llm_benchmark.go | 60 + .../modules/llm/mod_llm_benchmark_package.go | 35 + pkg/mcclient/options/llm/image.go | 6 +- pkg/mcclient/options/llm/llm_benchmark.go | 134 ++ .../options/llm/llm_benchmark_package.go | 73 + 27 files changed, 5342 insertions(+), 3 deletions(-) create mode 100644 cmd/climc/shell/llm/llm_benchmark.go create mode 100644 cmd/climc/shell/llm/llm_benchmark_package.go create mode 100644 pkg/apigateway/handler/llm_benchmark_artifact.go create mode 100644 pkg/apis/llm/llm_benchmark.go create mode 100644 pkg/apis/llm/llm_benchmark_package.go create mode 100644 pkg/llm/benchmark/artifact_store.go create mode 100644 pkg/llm/benchmark/doc.go create mode 100644 pkg/llm/benchmark/evaluation.go create mode 100644 pkg/llm/benchmark/guidellm.go create mode 100644 pkg/llm/models/llm_benchmark.go create mode 100644 pkg/llm/models/llm_benchmark_package.go create mode 100644 pkg/llm/models/llm_benchmark_stop_state.go create mode 100644 pkg/llm/service/benchmark_handler.go create mode 100644 pkg/llm/tasks/llm/llm_benchmark_delete_task.go create mode 100644 pkg/llm/tasks/llm/llm_benchmark_package_delete_task.go create mode 100644 pkg/llm/tasks/llm/llm_benchmark_package_import_task.go create mode 100644 pkg/llm/tasks/llm/llm_benchmark_run_task.go create mode 100644 pkg/mcclient/modules/llm/mod_llm_benchmark.go create mode 100644 pkg/mcclient/modules/llm/mod_llm_benchmark_package.go create mode 100644 pkg/mcclient/options/llm/llm_benchmark.go create mode 100644 pkg/mcclient/options/llm/llm_benchmark_package.go diff --git a/cmd/climc/shell/llm/llm_benchmark.go b/cmd/climc/shell/llm/llm_benchmark.go new file mode 100644 index 0000000000..f5f5ed02d6 --- /dev/null +++ b/cmd/climc/shell/llm/llm_benchmark.go @@ -0,0 +1,55 @@ +package llm + +import ( + "io" + "os" + + "github.com/cheggaaa/pb/v3" + + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/cmd/climc/shell" + "yunion.io/x/onecloud/pkg/mcclient" + modules "yunion.io/x/onecloud/pkg/mcclient/modules/llm" + options "yunion.io/x/onecloud/pkg/mcclient/options/llm" +) + +func init() { + cmd := shell.NewResourceCmd(&modules.LLMBenchmarks) + cmd.List(new(options.LLMBenchmarkListOptions)) + cmd.Show(new(options.LLMBenchmarkShowOptions)) + cmd.Create(new(options.LLMBenchmarkCreateOptions)) + cmd.Update(new(options.LLMBenchmarkUpdateOptions)) + cmd.Delete(new(options.LLMBenchmarkDeleteOptions)) + cmd.Perform("copy", new(options.LLMBenchmarkCopyOptions)) + cmd.Perform("retest", new(options.LLMBenchmarkRetestOptions)) + cmd.Perform("stop", new(options.LLMBenchmarkStopOptions)) + + shell.R(new(options.LLMBenchmarkArtifactOptions), "llm-benchmark-artifact", "Download benchmark artifact", func(s *mcclient.ClientSession, args *options.LLMBenchmarkArtifactOptions) error { + src, size, err := modules.LLMBenchmarks.Artifact(s, args.ID, args.Type) + if err != nil { + return errors.Wrap(err, "download artifact") + } + defer src.Close() + return writeBenchmarkArtifact(src, size, args.Output) + }) +} + +func writeBenchmarkArtifact(src io.Reader, size int64, output string) error { + if output == "" { + _, err := io.Copy(os.Stdout, src) + return err + } + f, err := os.Create(output) + if err != nil { + return err + } + defer f.Close() + if size < 0 { + _, err = io.Copy(f, src) + return err + } + bar := pb.Full.Start64(size) + _, err = io.Copy(f, bar.NewProxyReader(src)) + return err +} diff --git a/cmd/climc/shell/llm/llm_benchmark_package.go b/cmd/climc/shell/llm/llm_benchmark_package.go new file mode 100644 index 0000000000..6f55c6d8a9 --- /dev/null +++ b/cmd/climc/shell/llm/llm_benchmark_package.go @@ -0,0 +1,16 @@ +package llm + +import ( + "yunion.io/x/onecloud/cmd/climc/shell" + modules "yunion.io/x/onecloud/pkg/mcclient/modules/llm" + options "yunion.io/x/onecloud/pkg/mcclient/options/llm" +) + +func init() { + cmd := shell.NewResourceCmd(&modules.LLMBenchmarkPackages) + cmd.List(new(options.LLMBenchmarkPackageListOptions)) + cmd.Show(new(options.LLMBenchmarkPackageShowOptions)) + cmd.Create(new(options.LLMBenchmarkPackageCreateOptions)) + cmd.Delete(new(options.LLMBenchmarkPackageDeleteOptions)) + cmd.PerformClass("import", new(options.LLMBenchmarkPackageImportOptions)) +} diff --git a/pkg/apigateway/handler/llm_benchmark_artifact.go b/pkg/apigateway/handler/llm_benchmark_artifact.go new file mode 100644 index 0000000000..df09430fff --- /dev/null +++ b/pkg/apigateway/handler/llm_benchmark_artifact.go @@ -0,0 +1,75 @@ +package handler + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + + "yunion.io/x/log" + "yunion.io/x/pkg/appctx" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient/auth" + llmmodules "yunion.io/x/onecloud/pkg/mcclient/modules/llm" +) + +func llmBenchmarkArtifactsHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) { + proxyLLMBenchmarkArtifact(ctx, w, r, "") +} + +func llmBenchmarkArtifactHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) { + kind := appctx.AppContextParams(ctx)[""] + if kind == "" { + httperrors.MissingParameterError(ctx, w, "type") + return + } + proxyLLMBenchmarkArtifact(ctx, w, r, kind) +} + +func proxyLLMBenchmarkArtifact(ctx context.Context, w http.ResponseWriter, r *http.Request, kind string) { + id := appctx.AppContextParams(ctx)[""] + path := llmBenchmarkArtifactsBackendPath(id) + if kind != "" { + path = llmBenchmarkArtifactBackendPath(id, kind) + } + + s := auth.GetSession(ctx, AppContextToken(ctx), FetchRegion(r)) + resp, err := s.RawVersionRequest( + llmmodules.LLMBenchmarks.ServiceType(), + llmmodules.LLMBenchmarks.EndpointType(), + "GET", + path, + nil, + nil, + ) + if err != nil { + httperrors.GeneralServerError(ctx, w, errors.Wrap(err, "request backend")) + return + } + defer resp.Body.Close() + + copyHTTPHeader(w.Header(), resp.Header) + w.WriteHeader(resp.StatusCode) + if _, err := io.Copy(w, resp.Body); err != nil { + log.Errorf("copy benchmark artifact response: %v", err) + } +} + +func llmBenchmarkArtifactsBackendPath(id string) string { + return fmt.Sprintf("/llm_benchmarks/%s/artifacts", url.PathEscape(id)) +} + +func llmBenchmarkArtifactBackendPath(id string, kind string) string { + return fmt.Sprintf("%s/%s", llmBenchmarkArtifactsBackendPath(id), url.PathEscape(kind)) +} + +func copyHTTPHeader(dst http.Header, src http.Header) { + for k, vv := range src { + for _, v := range vv { + dst.Add(k, v) + } + } +} diff --git a/pkg/apigateway/handler/misc.go b/pkg/apigateway/handler/misc.go index ff3a832523..02e4dddf09 100644 --- a/pkg/apigateway/handler/misc.go +++ b/pkg/apigateway/handler/misc.go @@ -122,6 +122,10 @@ func (h *MiscHandler) Bind(app *appsrv.Application) { app.AddHandler(GET, prefix+"mcp-servers-config", mcpServersConfigHandler) // mcp agent default MCP server tools (options.MCPServerURL only, no mcp_agent entry) app.AddHandler(GET, prefix+"default-mcp-tools", FetchAuthToken(mcpAgentDefaultToolsHandler)) + + benchArtifactPrefix := prefix + "llm_benchmarks//artifacts" + app.AddHandler(GET, benchArtifactPrefix, FetchAuthToken(llmBenchmarkArtifactsHandler)) + app.AddHandler(GET, benchArtifactPrefix+"/", FetchAuthToken(llmBenchmarkArtifactHandler)) } func UploadHandlerInfo(method, prefix string, handler func(context.Context, http.ResponseWriter, *http.Request)) *appsrv.SHandlerInfo { diff --git a/pkg/apis/llm/image.go b/pkg/apis/llm/image.go index bdfce3968e..1221935b36 100644 --- a/pkg/apis/llm/image.go +++ b/pkg/apis/llm/image.go @@ -18,6 +18,7 @@ const ( LLM_IMAGE_TYPE_HERMES_AGENT LLMImageType = "hermes-agent" LLM_IMAGE_TYPE_LLM_ROUTER LLMImageType = "llm-router" LLM_IMAGE_TYPE_DESKTOP LLMImageType = "desktop" + LLM_IMAGE_TYPE_BENCHMARK LLMImageType = "benchmark" ) var ( @@ -31,6 +32,7 @@ var ( string(LLM_IMAGE_TYPE_HERMES_AGENT), string(LLM_IMAGE_TYPE_LLM_ROUTER), string(LLM_IMAGE_TYPE_DESKTOP), + string(LLM_IMAGE_TYPE_BENCHMARK), ) ) diff --git a/pkg/apis/llm/llm_benchmark.go b/pkg/apis/llm/llm_benchmark.go new file mode 100644 index 0000000000..0c7b91f779 --- /dev/null +++ b/pkg/apis/llm/llm_benchmark.go @@ -0,0 +1,166 @@ +package llm + +import ( + "reflect" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/gotypes" + + "yunion.io/x/onecloud/pkg/apis" +) + +const ( + LLMBenchmarkStatePending = "pending" + LLMBenchmarkStateQueued = "queued" + LLMBenchmarkStateValidating = "validating" + LLMBenchmarkStateRunning = "running" + LLMBenchmarkStateCompleted = "completed" + LLMBenchmarkStateStopped = "stopped" + LLMBenchmarkStateError = "error" +) + +const ( + LLMBenchmarkDatasetSyntheticText = "synthetic_text" + LLMBenchmarkDatasetPackage = "benchmark_package" + LLMBenchmarkProfileConstant = "constant" +) + +const ( + LLMBenchmarkEvaluationStateEvaluating = "evaluating" + LLMBenchmarkEvaluationStateCompleted = "completed" + LLMBenchmarkEvaluationStateSkipped = "skipped" + LLMBenchmarkEvaluationStateError = "error" +) + +const ( + LLMBenchmarkArtifactStorageLocal = "local" + LLMBenchmarkArtifactStorageMinio = "minio" +) + +const ( + LLMBenchmarkDefaultRequestFormat = "/v1/chat/completions" + LLMBenchmarkDefaultImage = "registry.cn-beijing.aliyuncs.com/cloudpods/guidellm:v0.7.0-amd64" +) + +type LLMBenchmarkCreateInput struct { + apis.VirtualResourceCreateInput + + LLMId string `json:"llm_id"` + BenchmarkImage string `json:"benchmark_image,omitempty"` + BenchmarkPackage string `json:"benchmark_package,omitempty"` + + RequestFormat string `json:"request_format,omitempty"` + Model string `json:"model,omitempty"` + + Profile string `json:"profile,omitempty"` + RequestRate int `json:"request_rate,omitempty"` + + TotalRequests int `json:"total_requests,omitempty"` + MaxDurationSeconds int `json:"max_duration_seconds,omitempty"` + MaxErrors int `json:"max_errors,omitempty"` + + DatasetName string `json:"dataset_name,omitempty"` + DatasetInputTokens int `json:"dataset_input_tokens,omitempty"` + DatasetOutputTokens int `json:"dataset_output_tokens,omitempty"` + DatasetPath string `json:"dataset_path,omitempty"` + + LLMDeploymentId string `json:"llm_deployment_id,omitempty"` + LLMSkuId string `json:"llm_sku_id,omitempty"` + LLMImageId string `json:"llm_image_id,omitempty"` + BenchmarkPackageId string `json:"benchmark_package_id,omitempty"` + Backend string `json:"backend,omitempty"` + TargetUrl string `json:"target_url,omitempty"` + WorkDir string `json:"work_dir,omitempty"` + TargetSnapshot string `json:"target_snapshot,omitempty"` + GuideLLMSpec string `json:"guide_llm_spec,omitempty"` +} + +type LLMBenchmarkCopyInput struct { + Name string `json:"name"` + LLMDeploymentId string `json:"llm_deployment_id"` + + Description *string `json:"description,omitempty"` + Model *string `json:"model,omitempty"` + RequestRate *int `json:"request_rate,omitempty"` + TotalRequests *int `json:"total_requests,omitempty"` + MaxDurationSeconds *int `json:"max_duration_seconds,omitempty"` + MaxErrors *int `json:"max_errors,omitempty"` + DatasetInputTokens *int `json:"dataset_input_tokens,omitempty"` + DatasetOutputTokens *int `json:"dataset_output_tokens,omitempty"` +} + +type LLMBenchmarkUpdateInput struct { + apis.VirtualResourceBaseUpdateInput + + Model *string `json:"model,omitempty"` + RequestRate *int `json:"request_rate,omitempty"` + TotalRequests *int `json:"total_requests,omitempty"` + MaxDurationSeconds *int `json:"max_duration_seconds,omitempty"` + MaxErrors *int `json:"max_errors,omitempty"` + DatasetInputTokens *int `json:"dataset_input_tokens,omitempty"` + DatasetOutputTokens *int `json:"dataset_output_tokens,omitempty"` +} + +type LLMBenchmarkRetestInput struct{} + +type LLMBenchmarkListInput struct { + apis.VirtualResourceListInput + + LLMId string `json:"llm_id"` + LLMDeploymentId string `json:"llm_deployment_id"` + State string `json:"state"` +} + +type LLMBenchmarkDatasetPreflight struct { + State string `json:"state"` + ExpectedSamples int `json:"expected_samples"` + ActualSamples int `json:"actual_samples"` + Successful int `json:"successful"` + Errored int `json:"errored"` + ErrorRate float64 `json:"error_rate"` + LatencyMeanSeconds float64 `json:"latency_mean_sec"` + Message string `json:"message"` +} + +func (p *LLMBenchmarkDatasetPreflight) String() string { + return jsonutils.Marshal(p).String() +} + +func (p *LLMBenchmarkDatasetPreflight) IsZero() bool { + return p == nil || *p == (LLMBenchmarkDatasetPreflight{}) +} + +type LLMBenchmarkDatasetEvaluation struct { + State string `json:"state"` + AnswerColumn string `json:"answer_column,omitempty"` + RequestTotal int `json:"request_total"` + Evaluated int `json:"evaluated"` + Correct int `json:"correct"` + Incorrect int `json:"incorrect"` + Unscored int `json:"unscored"` + Accuracy float64 `json:"accuracy"` + Message string `json:"message,omitempty"` +} + +func (e *LLMBenchmarkDatasetEvaluation) String() string { + return jsonutils.Marshal(e).String() +} + +func (e *LLMBenchmarkDatasetEvaluation) IsZero() bool { + return e == nil || *e == (LLMBenchmarkDatasetEvaluation{}) +} + +func init() { + gotypes.RegisterSerializable(reflect.TypeOf(new(LLMBenchmarkDatasetPreflight)), func() gotypes.ISerializable { + return new(LLMBenchmarkDatasetPreflight) + }) + gotypes.RegisterSerializable(reflect.TypeOf(new(LLMBenchmarkDatasetEvaluation)), func() gotypes.ISerializable { + return new(LLMBenchmarkDatasetEvaluation) + }) +} + +type LLMBenchmarkDetails struct { + apis.VirtualResourceDetails + + LLMDeployment string `json:"llm_deployment"` +} diff --git a/pkg/apis/llm/llm_benchmark_package.go b/pkg/apis/llm/llm_benchmark_package.go new file mode 100644 index 0000000000..c007f887d0 --- /dev/null +++ b/pkg/apis/llm/llm_benchmark_package.go @@ -0,0 +1,42 @@ +package llm + +import "yunion.io/x/onecloud/pkg/apis" + +const ( + LLMBenchmarkPackageSourceHuggingFace = "huggingface" + LLMBenchmarkPackageFormatGuideLLMJSONL = "guidellm_jsonl" + + LLMBenchmarkPackageMountBase = "/data/benchmark-packages" +) + +type LLMBenchmarkPackageCreateInput struct { + apis.SharableVirtualResourceCreateInput + + Source string `json:"source,omitempty"` + RepoId string `json:"repo_id,omitempty"` + Revision string `json:"revision,omitempty"` + FilePath string `json:"file_path,omitempty"` + Format string `json:"format,omitempty"` + AnswerColumn string `json:"answer_column,omitempty"` + + ImageId string `json:"image_id,omitempty"` + Size int64 `json:"size,omitempty"` + ActualSizeMb int32 `json:"actual_size_mb,omitempty"` + + MountPath string `json:"mount_path,omitempty"` + DatasetPath string `json:"dataset_path,omitempty"` + Manifest string `json:"manifest,omitempty"` +} + +type LLMBenchmarkPackageImportInput struct { + LLMBenchmarkPackageCreateInput + BenchmarkSpec *LLMBenchmarkCreateInput `json:"benchmark_spec,omitempty"` +} + +type LLMBenchmarkPackageListInput struct { + apis.SharableVirtualResourceListInput + + Source string `json:"source"` + RepoId string `json:"repo_id"` + Format string `json:"format"` +} diff --git a/pkg/llm/benchmark/artifact_store.go b/pkg/llm/benchmark/artifact_store.go new file mode 100644 index 0000000000..7d9455f3ca --- /dev/null +++ b/pkg/llm/benchmark/artifact_store.go @@ -0,0 +1,286 @@ +package benchmark + +import ( + "context" + "errors" + "io" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "sort" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + smithyhttp "github.com/aws/smithy-go/transport/http" + + api "yunion.io/x/onecloud/pkg/apis/llm" +) + +type ArtifactStoreOptions struct { + Endpoint string + AccessKey string + SecretKey string + Bucket string + Secure bool + Prefix string +} + +type ArtifactStore struct { + opts ArtifactStoreOptions + client *s3.Client + initErr error +} + +var defaultArtifactStore = NewArtifactStore(ArtifactStoreOptions{}) + +func ConfigureArtifactStore(opts ArtifactStoreOptions) { + defaultArtifactStore = NewArtifactStore(opts) +} + +func DefaultArtifactStore() *ArtifactStore { + return defaultArtifactStore +} + +func NewArtifactStore(opts ArtifactStoreOptions) *ArtifactStore { + opts.Endpoint = strings.TrimSpace(opts.Endpoint) + opts.Prefix = strings.Trim(strings.TrimSpace(opts.Prefix), "/") + ret := &ArtifactStore{opts: opts} + if opts.Endpoint == "" { + return ret + } + if opts.AccessKey == "" || opts.SecretKey == "" || opts.Bucket == "" { + ret.initErr = errors.New("missing MinIO/S3 artifact config") + return ret + } + endpoint := opts.Endpoint + if !strings.HasPrefix(endpoint, "http://") && !strings.HasPrefix(endpoint, "https://") { + if opts.Secure { + endpoint = "https://" + endpoint + } else { + endpoint = "http://" + endpoint + } + } + ret.client = s3.NewFromConfig(aws.Config{ + Region: "us-east-1", + Credentials: credentials.NewStaticCredentialsProvider(opts.AccessKey, opts.SecretKey, ""), + BaseEndpoint: aws.String(endpoint), + }, func(o *s3.Options) { + o.UsePathStyle = true + }) + return ret +} + +func (s *ArtifactStore) Enabled() bool { + return s != nil && s.opts.Endpoint != "" +} + +func (s *ArtifactStore) benchmarkPrefix(projectID, benchmarkID string) string { + return path.Join(s.opts.Prefix, projectID, benchmarkID) + "/" +} + +func (s *ArtifactStore) objectKey(projectID, benchmarkID, local string) string { + return path.Join(s.opts.Prefix, projectID, benchmarkID, filepath.Base(local)) +} + +func artifactURI(bucket, key string) string { + return "s3://" + bucket + "/" + key +} + +func parseArtifactURI(location string) (string, string, error) { + parsed, err := url.Parse(location) + if err != nil || parsed.Scheme != "s3" || parsed.Host == "" { + return "", "", errors.New("invalid artifact S3 URI") + } + key := strings.TrimPrefix(parsed.EscapedPath(), "/") + key, err = url.PathUnescape(key) + if err != nil || key == "" { + return "", "", errors.New("invalid artifact S3 object key") + } + return parsed.Host, key, nil +} + +func cloneArtifactPaths(paths map[string]string) map[string]string { + ret := make(map[string]string, len(paths)) + for kind, location := range paths { + if location != "" { + ret[kind] = location + } + } + return ret +} + +func artifactNotFound(err error) bool { + var responseError *smithyhttp.ResponseError + return errors.As(err, &responseError) && responseError.HTTPStatusCode() == http.StatusNotFound +} + +func (s *ArtifactStore) Persist( + ctx context.Context, + projectID, benchmarkID string, + files map[string]string, +) (map[string]string, string, error) { + local := cloneArtifactPaths(files) + if len(local) == 0 { + return local, "", nil + } + if !s.Enabled() { + return local, api.LLMBenchmarkArtifactStorageLocal, nil + } + if s.initErr != nil { + return local, api.LLMBenchmarkArtifactStorageLocal, s.initErr + } + if err := ensureArtifactBucket(ctx, s.client, s.opts.Bucket); err != nil { + return local, api.LLMBenchmarkArtifactStorageLocal, err + } + + kinds := make([]string, 0, len(local)) + for kind := range local { + kinds = append(kinds, kind) + } + sort.Strings(kinds) + remote := make(map[string]string, len(local)) + uploaded := make([]string, 0, len(local)) + for _, kind := range kinds { + file, err := os.Open(local[kind]) + if err == nil { + key := s.objectKey(projectID, benchmarkID, local[kind]) + uploaded = append(uploaded, key) + _, err = s.client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(s.opts.Bucket), + Key: aws.String(key), + Body: file, + }) + closeErr := file.Close() + if err == nil { + err = closeErr + } + if err == nil { + remote[kind] = artifactURI(s.opts.Bucket, key) + } + } + if err != nil { + for _, key := range uploaded { + _, _ = s.client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(s.opts.Bucket), + Key: aws.String(key), + }) + } + return local, api.LLMBenchmarkArtifactStorageLocal, err + } + } + return remote, api.LLMBenchmarkArtifactStorageMinio, nil +} + +func ensureArtifactBucket(ctx context.Context, client *s3.Client, bucket string) error { + if _, err := client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: aws.String(bucket)}); err == nil { + return nil + } + _, err := client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(bucket)}) + return err +} + +func (s *ArtifactStore) RemoveLocal(files map[string]string) error { + var firstErr error + for _, file := range files { + if file == "" { + continue + } + if err := os.Remove(file); err != nil && !os.IsNotExist(err) && firstErr == nil { + firstErr = err + } + } + return firstErr +} + +func (s *ArtifactStore) Exists(ctx context.Context, location string) (bool, error) { + if location == "" { + return false, nil + } + if !strings.HasPrefix(location, "s3://") { + _, err := os.Stat(location) + if os.IsNotExist(err) { + return false, nil + } + return err == nil, err + } + if s == nil || s.client == nil { + if s != nil && s.initErr != nil { + return false, s.initErr + } + return false, errors.New("MinIO/S3 artifact store is disabled") + } + bucket, key, err := parseArtifactURI(location) + if err != nil { + return false, err + } + _, err = s.client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + }) + if artifactNotFound(err) { + return false, nil + } + return err == nil, err +} + +func (s *ArtifactStore) Open(ctx context.Context, location string) (io.ReadCloser, error) { + if !strings.HasPrefix(location, "s3://") { + return os.Open(location) + } + if s == nil || s.client == nil { + if s != nil && s.initErr != nil { + return nil, s.initErr + } + return nil, errors.New("MinIO/S3 artifact store is disabled") + } + bucket, key, err := parseArtifactURI(location) + if err != nil { + return nil, err + } + out, err := s.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + }) + if err != nil { + return nil, err + } + return out.Body, nil +} + +func (s *ArtifactStore) DeleteBenchmark(ctx context.Context, projectID, benchmarkID string) error { + if !s.Enabled() { + return nil + } + if s.initErr != nil { + return s.initErr + } + input := &s3.ListObjectsV2Input{ + Bucket: aws.String(s.opts.Bucket), + Prefix: aws.String(s.benchmarkPrefix(projectID, benchmarkID)), + } + for { + out, err := s.client.ListObjectsV2(ctx, input) + if artifactNotFound(err) { + return nil + } + if err != nil { + return err + } + for _, object := range out.Contents { + if _, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(s.opts.Bucket), + Key: object.Key, + }); err != nil { + return err + } + } + if !aws.ToBool(out.IsTruncated) || out.NextContinuationToken == nil { + return nil + } + input.ContinuationToken = out.NextContinuationToken + } +} diff --git a/pkg/llm/benchmark/doc.go b/pkg/llm/benchmark/doc.go new file mode 100644 index 0000000000..bef76c1743 --- /dev/null +++ b/pkg/llm/benchmark/doc.go @@ -0,0 +1 @@ +package benchmark // import "yunion.io/x/onecloud/pkg/llm/benchmark" diff --git a/pkg/llm/benchmark/evaluation.go b/pkg/llm/benchmark/evaluation.go new file mode 100644 index 0000000000..11ee1197cf --- /dev/null +++ b/pkg/llm/benchmark/evaluation.go @@ -0,0 +1,772 @@ +package benchmark + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/csv" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "math/big" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "unicode" + "unicode/utf8" + + api "yunion.io/x/onecloud/pkg/apis/llm" +) + +const evaluationExcerptRunes = 1024 + +var answerColumnCandidates = []string{ + "reference_answer", + "answer", + "solution", + "target", + "label", +} + +var promptColumnCandidates = []string{ + "prompt", + "instruction", + "question", + "input", + "context", + "content", + "conversation", + "turn", + "text", +} + +var decimalNumberPattern = regexp.MustCompile(`^[+-]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][+-]?[0-9]+)?$`) + +type EvaluationInput struct { + DatasetPath string + BenchmarkPath string + AnswerColumn string + OutputDir string +} + +type EvaluationResult struct { + Summary api.LLMBenchmarkDatasetEvaluation + ResultJSON string + ResultCSV string + LogPath string +} + +type evaluationExcerpt struct { + Text string `json:"text"` + RuneCount int `json:"rune_count"` + SHA256 string `json:"sha256"` + Truncated bool `json:"truncated"` +} + +type answerValue struct { + scorable bool + key string + textHash string + number string + excerpt evaluationExcerpt +} + +type datasetEntry struct { + index int + answer answerValue + conflict bool +} + +const ( + evaluationItemCorrect = "correct" + evaluationItemIncorrect = "incorrect" + evaluationItemUnscored = "unscored" +) + +type evaluationItem struct { + RequestID string `json:"request_id"` + DatasetIndex *int `json:"dataset_index,omitempty"` + State string `json:"state"` + MatchType string `json:"match_type,omitempty"` + ReferenceExcerpt *evaluationExcerpt `json:"reference_excerpt,omitempty"` + PromptExcerpt *evaluationExcerpt `json:"prompt_excerpt,omitempty"` + OutputExcerpt *evaluationExcerpt `json:"output_excerpt,omitempty"` + Message string `json:"message,omitempty"` +} + +var guideRequestArrayStates = map[string]string{ + "successful_requests": "successful", + "successful": "successful", + "errored_requests": "errored", + "errored": "errored", + "incomplete_requests": "incomplete", + "incomplete": "incomplete", + "timed_out_requests": "timed_out", + "timeout_requests": "timed_out", +} + +func normalizeWhitespace(value string) string { + value = strings.TrimSpace(value) + var out strings.Builder + pendingSpace := false + for _, r := range value { + if unicode.IsSpace(r) { + pendingSpace = true + continue + } + if pendingSpace && out.Len() > 0 { + out.WriteByte(' ') + } + pendingSpace = false + out.WriteRune(r) + } + return out.String() +} + +func canonicalFold(value string) string { + var out strings.Builder + for _, r := range value { + min := r + for next := unicode.SimpleFold(r); next != r; next = unicode.SimpleFold(next) { + if next < min { + min = next + } + } + out.WriteRune(min) + } + return out.String() +} + +func sha256String(value string) string { + sum := sha256.Sum256([]byte(value)) + return hex.EncodeToString(sum[:]) +} + +func parseExactNumber(value string) (string, bool) { + if !decimalNumberPattern.MatchString(value) { + return "", false + } + number, ok := new(big.Rat).SetString(value) + if !ok { + return "", false + } + return number.RatString(), true +} + +func makeEvaluationExcerpt(value string) evaluationExcerpt { + runeCount := utf8.RuneCountInString(value) + text := value + truncated := runeCount > evaluationExcerptRunes + if truncated { + text = string([]rune(value)[:evaluationExcerptRunes]) + } + return evaluationExcerpt{ + Text: text, + RuneCount: runeCount, + SHA256: sha256String(value), + Truncated: truncated, + } +} + +func scalarText(raw json.RawMessage) (string, bool) { + if len(raw) == 0 { + return "", false + } + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.UseNumber() + var value interface{} + if err := decoder.Decode(&value); err != nil { + return "", false + } + switch value := value.(type) { + case string: + return value, true + case json.Number: + return value.String(), true + case bool: + return strconv.FormatBool(value), true + default: + return "", false + } +} + +func newAnswerValue(raw json.RawMessage) answerValue { + text, ok := scalarText(raw) + if !ok { + return answerValue{key: "unscored"} + } + normalized := normalizeWhitespace(text) + ret := answerValue{ + scorable: true, + textHash: sha256String(canonicalFold(normalized)), + excerpt: makeEvaluationExcerpt(text), + } + if number, ok := parseExactNumber(normalized); ok { + ret.number = number + ret.key = "number:" + number + } else { + ret.key = "text:" + ret.textHash + } + return ret +} + +func matchAnswer(reference answerValue, output string) (bool, string) { + normalized := normalizeWhitespace(output) + if reference.number != "" { + if number, ok := parseExactNumber(normalized); ok { + return number == reference.number, "number" + } + } + return sha256String(canonicalFold(normalized)) == reference.textHash, "text" +} + +func forEachDatasetRow( + ctx context.Context, + path string, + visit func(index int, row map[string]json.RawMessage) error, +) error { + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + decoder := json.NewDecoder(file) + for index := 0; ; index++ { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + row := map[string]json.RawMessage{} + if err := decoder.Decode(&row); err != nil { + if errors.Is(err, io.EOF) { + return nil + } + return fmt.Errorf("decode dataset row %d: %w", index, err) + } + if err := visit(index, row); err != nil { + return err + } + } +} + +func detectAnswerColumn(ctx context.Context, path, explicit string) (string, string, error) { + explicit = strings.TrimSpace(explicit) + if explicit != "" { + found := false + err := forEachDatasetRow(ctx, path, func(_ int, row map[string]json.RawMessage) error { + if _, ok := row[explicit]; ok { + found = true + } + return nil + }) + if err != nil { + return "", "", err + } + if !found { + return "", fmt.Sprintf("answer column %q not found", explicit), nil + } + return explicit, "", nil + } + + found := map[string]bool{} + err := forEachDatasetRow(ctx, path, func(_ int, row map[string]json.RawMessage) error { + for _, candidate := range answerColumnCandidates { + if _, ok := row[candidate]; ok { + found[candidate] = true + } + } + return nil + }) + if err != nil { + return "", "", err + } + matches := make([]string, 0, len(found)) + for _, candidate := range answerColumnCandidates { + if found[candidate] { + matches = append(matches, candidate) + } + } + switch len(matches) { + case 0: + return "", "no answer column found", nil + case 1: + return matches[0], "", nil + default: + return "", "multiple answer columns found: " + strings.Join(matches, ", "), nil + } +} + +func firstStringField(row map[string]json.RawMessage, candidates []string) (string, bool) { + for _, candidate := range candidates { + raw, ok := row[candidate] + if !ok { + continue + } + var value string + if json.Unmarshal(raw, &value) == nil { + return value, true + } + } + return "", false +} + +func promptDigest(prompt string) string { + return sha256String(normalizeWhitespace(prompt)) +} + +func buildDatasetIndex(ctx context.Context, path, answerColumn string) (map[string]*datasetEntry, error) { + // ponytail: the index is bounded by total_requests; move it to disk only if the 100k-request ceiling is proven too large. + index := map[string]*datasetEntry{} + err := forEachDatasetRow(ctx, path, func(rowIndex int, row map[string]json.RawMessage) error { + prompt, ok := firstStringField(row, promptColumnCandidates) + if !ok { + return nil + } + answer := newAnswerValue(row[answerColumn]) + key := promptDigest(prompt) + if existing := index[key]; existing != nil { + if existing.answer.key != answer.key { + existing.conflict = true + } + return nil + } + index[key] = &datasetEntry{ + index: rowIndex, + answer: answer, + } + return nil + }) + return index, err +} + +func walkGuideLLMRequests( + ctx context.Context, + path string, + visit func(state string, raw json.RawMessage) error, +) error { + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + decoder := json.NewDecoder(file) + return walkGuideLLMValue(ctx, decoder, "", visit) +} + +func walkGuideLLMValue( + ctx context.Context, + decoder *json.Decoder, + arrayState string, + visit func(state string, raw json.RawMessage) error, +) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + token, err := decoder.Token() + if err != nil { + return err + } + delimiter, ok := token.(json.Delim) + if !ok { + return nil + } + if arrayState != "" && delimiter != '[' { + arrayState = "" + } + switch delimiter { + case '{': + for decoder.More() { + keyToken, err := decoder.Token() + if err != nil { + return err + } + key, ok := keyToken.(string) + if !ok { + return errors.New("GuideLLM object key is not a string") + } + if err := walkGuideLLMValue(ctx, decoder, guideRequestArrayStates[key], visit); err != nil { + return err + } + } + _, err = decoder.Token() + return err + case '[': + if arrayState != "" { + for decoder.More() { + var raw json.RawMessage + if err := decoder.Decode(&raw); err != nil { + return err + } + if err := visit(arrayState, raw); err != nil { + return err + } + } + _, err = decoder.Token() + return err + } + for decoder.More() { + if err := walkGuideLLMValue(ctx, decoder, "", visit); err != nil { + return err + } + } + _, err = decoder.Token() + return err + default: + return fmt.Errorf("unexpected GuideLLM delimiter %q", delimiter) + } +} + +func decodeRawObject(raw json.RawMessage) map[string]json.RawMessage { + var object map[string]json.RawMessage + if json.Unmarshal(raw, &object) == nil { + return object + } + var encoded string + if json.Unmarshal(raw, &encoded) == nil && json.Unmarshal([]byte(encoded), &object) == nil { + return object + } + return nil +} + +func rawString(raw json.RawMessage) (string, bool) { + var value string + if json.Unmarshal(raw, &value) != nil { + return "", false + } + return value, true +} + +func extractPromptObject(object map[string]json.RawMessage) (string, bool) { + if object == nil { + return "", false + } + if prompt, ok := firstStringField(object, promptColumnCandidates); ok { + return prompt, true + } + for _, key := range []string{"body", "payload", "request"} { + if nested := decodeRawObject(object[key]); nested != nil { + if prompt, ok := extractPromptObject(nested); ok { + return prompt, true + } + } + } + var messages []map[string]json.RawMessage + if json.Unmarshal(object["messages"], &messages) != nil { + return "", false + } + userPrompt := "" + userCount := 0 + for _, message := range messages { + role, _ := rawString(message["role"]) + content, ok := rawString(message["content"]) + if role == "user" && ok { + userPrompt = content + userCount++ + } + } + return userPrompt, userCount == 1 +} + +func requestPrompt(object map[string]json.RawMessage) (string, bool) { + for _, key := range []string{"request_args", "request", "input"} { + if nested := decodeRawObject(object[key]); nested != nil { + if prompt, ok := extractPromptObject(nested); ok { + return prompt, true + } + } + } + return extractPromptObject(object) +} + +func extractOutputRaw(raw json.RawMessage) (string, bool) { + object := decodeRawObject(raw) + if object == nil { + return rawString(raw) + } + for _, key := range []string{"content", "text", "output", "generated_text"} { + if value, ok := rawString(object[key]); ok { + return value, true + } + } + for _, key := range []string{"message", "body", "response"} { + if value, ok := extractOutputRaw(object[key]); ok { + return value, true + } + } + var choices []map[string]json.RawMessage + if json.Unmarshal(object["choices"], &choices) == nil { + for _, choice := range choices { + if value, ok := extractOutputRaw(choice["message"]); ok { + return value, true + } + if value, ok := rawString(choice["text"]); ok { + return value, true + } + } + } + return "", false +} + +func requestOutput(object map[string]json.RawMessage) (string, bool) { + for _, key := range []string{"response", "response_args", "output", "result"} { + if value, ok := extractOutputRaw(object[key]); ok { + return value, true + } + } + return "", false +} + +func requestID(object map[string]json.RawMessage, ordinal int) string { + for _, key := range []string{"id", "request_id", "request_uuid"} { + if value, ok := rawString(object[key]); ok && value != "" { + return value + } + } + return fmt.Sprintf("request-%d", ordinal) +} + +func intPointer(value int) *int { + return &value +} + +func scoreGuideRequest( + state string, + raw json.RawMessage, + index map[string]*datasetEntry, + ordinal int, +) evaluationItem { + object := decodeRawObject(raw) + item := evaluationItem{RequestID: requestID(object, ordinal)} + prompt, ok := requestPrompt(object) + if !ok { + item.State = evaluationItemUnscored + item.Message = "single-turn prompt not found in request_args" + return item + } + promptExcerpt := makeEvaluationExcerpt(prompt) + item.PromptExcerpt = &promptExcerpt + entry := index[promptDigest(prompt)] + if entry == nil { + item.State = evaluationItemUnscored + item.Message = "prompt not found in evaluation dataset" + return item + } + item.DatasetIndex = intPointer(entry.index) + item.ReferenceExcerpt = &entry.answer.excerpt + if entry.conflict { + item.State = evaluationItemUnscored + item.Message = "duplicate prompt has conflicting reference answers" + return item + } + if !entry.answer.scorable { + item.State = evaluationItemUnscored + item.ReferenceExcerpt = nil + item.Message = "reference answer is missing or not a scalar" + return item + } + if state != "successful" { + item.State = evaluationItemIncorrect + item.Message = "request " + state + return item + } + output, ok := requestOutput(object) + if !ok || normalizeWhitespace(output) == "" { + item.State = evaluationItemIncorrect + item.Message = "successful request has empty output" + return item + } + outputExcerpt := makeEvaluationExcerpt(output) + item.OutputExcerpt = &outputExcerpt + matched, matchType := matchAnswer(entry.answer, output) + item.MatchType = matchType + if matched { + item.State = evaluationItemCorrect + } else { + item.State = evaluationItemIncorrect + } + return item +} + +func addEvaluationItem(summary *api.LLMBenchmarkDatasetEvaluation, item evaluationItem) { + summary.RequestTotal++ + switch item.State { + case evaluationItemCorrect: + summary.Correct++ + case evaluationItemIncorrect: + summary.Incorrect++ + case evaluationItemUnscored: + summary.Unscored++ + } +} + +var evaluationCSVHeader = []string{ + "request_id", "dataset_index", "state", "match_type", "message", + "prompt_excerpt", "prompt_rune_count", "prompt_sha256", "prompt_truncated", + "reference_excerpt", "reference_rune_count", "reference_sha256", "reference_truncated", + "output_excerpt", "output_rune_count", "output_sha256", "output_truncated", +} + +func excerptCSV(value *evaluationExcerpt) []string { + if value == nil { + return []string{"", "", "", ""} + } + return []string{ + value.Text, + strconv.Itoa(value.RuneCount), + value.SHA256, + strconv.FormatBool(value.Truncated), + } +} + +func evaluationCSVRow(item evaluationItem) []string { + datasetIndex := "" + if item.DatasetIndex != nil { + datasetIndex = strconv.Itoa(*item.DatasetIndex) + } + row := []string{item.RequestID, datasetIndex, item.State, item.MatchType, item.Message} + row = append(row, excerptCSV(item.PromptExcerpt)...) + row = append(row, excerptCSV(item.ReferenceExcerpt)...) + row = append(row, excerptCSV(item.OutputExcerpt)...) + return row +} + +func EvaluateDataset(ctx context.Context, input EvaluationInput) (ret EvaluationResult, retErr error) { + if err := os.MkdirAll(input.OutputDir, 0755); err != nil { + return ret, err + } + ret.LogPath = filepath.Join(input.OutputDir, "evaluation.log") + logFile, err := os.Create(ret.LogPath) + if err != nil { + return ret, err + } + defer logFile.Close() + defer func() { + if retErr == nil { + return + } + ret.Summary.State = api.LLMBenchmarkEvaluationStateError + ret.Summary.Message = retErr.Error() + _, _ = fmt.Fprintf(logFile, "evaluation error: %s\n", retErr) + }() + + column, skipMessage, err := detectAnswerColumn(ctx, input.DatasetPath, input.AnswerColumn) + if err != nil { + return ret, err + } + if skipMessage != "" { + ret.Summary.State = api.LLMBenchmarkEvaluationStateSkipped + ret.Summary.Message = skipMessage + _, _ = fmt.Fprintf(logFile, "evaluation skipped: %s\n", skipMessage) + return ret, nil + } + ret.Summary.AnswerColumn = column + _, _ = fmt.Fprintf(logFile, "answer column: %s\n", column) + + index, err := buildDatasetIndex(ctx, input.DatasetPath, column) + if err != nil { + return ret, err + } + + jsonPath := filepath.Join(input.OutputDir, "evaluation.json") + csvPath := filepath.Join(input.OutputDir, "evaluation.csv") + jsonFile, err := os.Create(jsonPath) + if err != nil { + return ret, err + } + csvFile, err := os.Create(csvPath) + if err != nil { + _ = jsonFile.Close() + _ = os.Remove(jsonPath) + return ret, err + } + reportsComplete := false + defer func() { + _ = jsonFile.Close() + _ = csvFile.Close() + if !reportsComplete { + _ = os.Remove(jsonPath) + _ = os.Remove(csvPath) + } + }() + + if _, err := io.WriteString(jsonFile, "{\"items\":["); err != nil { + return ret, err + } + jsonEncoder := json.NewEncoder(jsonFile) + csvWriter := csv.NewWriter(csvFile) + if err := csvWriter.Write(evaluationCSVHeader); err != nil { + return ret, err + } + first := true + ordinal := 0 + err = walkGuideLLMRequests(ctx, input.BenchmarkPath, func(state string, raw json.RawMessage) error { + ordinal++ + item := scoreGuideRequest(state, raw, index, ordinal) + addEvaluationItem(&ret.Summary, item) + if !first { + if _, err := io.WriteString(jsonFile, ","); err != nil { + return err + } + } + first = false + if err := jsonEncoder.Encode(item); err != nil { + return err + } + return csvWriter.Write(evaluationCSVRow(item)) + }) + if err != nil { + return ret, err + } + if ret.Summary.RequestTotal == 0 { + err = errors.New("no GuideLLM request records found") + return ret, err + } + + ret.Summary.Evaluated = ret.Summary.Correct + ret.Summary.Incorrect + if ret.Summary.Evaluated > 0 { + ret.Summary.Accuracy = float64(ret.Summary.Correct) / float64(ret.Summary.Evaluated) + } else { + ret.Summary.Message = "no scorable requests" + } + ret.Summary.State = api.LLMBenchmarkEvaluationStateCompleted + if _, err := io.WriteString(jsonFile, "],\"summary\":"); err != nil { + return ret, err + } + if err := jsonEncoder.Encode(ret.Summary); err != nil { + return ret, err + } + if _, err := io.WriteString(jsonFile, "}"); err != nil { + return ret, err + } + csvWriter.Flush() + if err := csvWriter.Error(); err != nil { + return ret, err + } + if err := jsonFile.Close(); err != nil { + return ret, err + } + if err := csvFile.Close(); err != nil { + return ret, err + } + reportsComplete = true + ret.ResultJSON = jsonPath + ret.ResultCSV = csvPath + _, _ = fmt.Fprintf( + logFile, + "requests=%d evaluated=%d correct=%d incorrect=%d unscored=%d accuracy=%g\n", + ret.Summary.RequestTotal, + ret.Summary.Evaluated, + ret.Summary.Correct, + ret.Summary.Incorrect, + ret.Summary.Unscored, + ret.Summary.Accuracy, + ) + return ret, nil +} diff --git a/pkg/llm/benchmark/guidellm.go b/pkg/llm/benchmark/guidellm.go new file mode 100644 index 0000000000..b4d2cb49d1 --- /dev/null +++ b/pkg/llm/benchmark/guidellm.go @@ -0,0 +1,404 @@ +package benchmark + +import ( + "encoding/csv" + "encoding/json" + "fmt" + "os" + "sort" + "strconv" + "strings" + + commonapi "yunion.io/x/onecloud/pkg/apis" + computeapi "yunion.io/x/onecloud/pkg/apis/compute" +) + +type GuideLLMSpecInput struct { + TargetURL string + RequestFormat string + Model string + RequestRate int + TotalRequests int + MaxDurationSeconds int + MaxErrors int + DatasetInputTokens int + DatasetOutputTokens int + DatasetPath string +} + +type GuideLLMSpec struct { + Backend map[string]interface{} `json:"backend"` + Profile map[string]interface{} `json:"profile"` + Tokenizer map[string]interface{} `json:"tokenizer,omitempty"` + Constraints []map[string]interface{} `json:"constraints"` + Data []map[string]interface{} `json:"data"` +} + +type LLMBenchmarkMetrics struct { + RequestsPerSecondMean *float64 + RequestLatencyMeanSec *float64 + RequestTotal int + RequestSuccessful int + RequestErrored int + ErrorRate *float64 +} + +type RunnerPodInput struct { + Name string + Image string + NetworkId string + HostId string + CPU int + MemoryMB int + PackageImageId string + PackageSizeMB int + PackageMountBase string + ModelImageId string + ModelSizeMB int + ModelMountBase string + ModelMountSubdirectory string +} + +const DatasetPreflightSamples = 10 + +func BuildGuideLLMSpec(input GuideLLMSpecInput) GuideLLMSpec { + backend := map[string]interface{}{ + "kind": "openai_http", + "target": input.TargetURL, + "request_format": input.RequestFormat, + } + if input.Model != "" { + backend["model"] = input.Model + } + tokenizer := map[string]interface{}{} + if strings.TrimSpace(input.DatasetPath) != "" && input.Model != "" { + tokenizer = map[string]interface{}{ + "kind": "hf_auto", + "model": tokenizerModel(input.Model), + } + } + data := []map[string]interface{}{ + { + "kind": "synthetic_text", + "prompt_tokens": input.DatasetInputTokens, + "output_tokens": input.DatasetOutputTokens, + }, + } + if strings.TrimSpace(input.DatasetPath) != "" { + data = []map[string]interface{}{ + { + "kind": "json_file", + "path": input.DatasetPath, + "load_kwargs": map[string]interface{}{ + "split": "train", + }, + }, + } + } + return GuideLLMSpec{ + Backend: backend, + Profile: map[string]interface{}{ + "kind": "constant", + "rate": input.RequestRate, + }, + Tokenizer: tokenizer, + Constraints: []map[string]interface{}{ + {"kind": "max_requests", "count": input.TotalRequests}, + {"kind": "max_duration", "seconds": input.MaxDurationSeconds}, + {"kind": "max_errors", "count": input.MaxErrors}, + }, + Data: data, + } +} + +func GuideLLMLocalTokenizer(modelPath string) map[string]interface{} { + return map[string]interface{}{ + "kind": "hf_auto", + "model": modelPath, + "load_kwargs": map[string]interface{}{ + "local_files_only": true, + }, + } +} + +func BuildGuideLLMPreflightSpec(formal GuideLLMSpec, maxDurationSeconds int) (GuideLLMSpec, error) { + if len(formal.Data) != 1 || formal.Data[0]["kind"] != "json_file" { + return GuideLLMSpec{}, fmt.Errorf("dataset preflight requires one json_file data source") + } + path, _ := formal.Data[0]["path"].(string) + if strings.TrimSpace(path) == "" { + return GuideLLMSpec{}, fmt.Errorf("dataset preflight path is empty") + } + data := map[string]interface{}{ + "kind": "json_file", + "path": path, + "load_kwargs": map[string]interface{}{ + "split": "train[:10]", + }, + } + return GuideLLMSpec{ + Backend: formal.Backend, + Tokenizer: formal.Tokenizer, + Profile: map[string]interface{}{ + "kind": "constant", + "rate": 1, + }, + Constraints: []map[string]interface{}{ + {"kind": "max_requests", "count": DatasetPreflightSamples}, + {"kind": "max_duration", "seconds": maxDurationSeconds}, + {"kind": "max_errors", "count": DatasetPreflightSamples}, + }, + Data: []map[string]interface{}{data}, + }, nil +} + +func GuideLLMRunCommand() []string { + return []string{ + "sh", + "-lc", + "mkdir -p /workdir && guidellm run --output kind=json,path=/workdir/benchmarks.json --output kind=csv,path=/workdir/benchmarks.csv > /workdir/guidellm.log 2>&1", + } +} + +func GuideLLMPreflightRunCommand(spec GuideLLMSpec) ([]string, error) { + envs, err := GuideLLMEnvs(spec) + if err != nil { + return nil, err + } + keys := make([]string, 0, len(envs)) + for key := range envs { + keys = append(keys, key) + } + sort.Strings(keys) + + parts := []string{"mkdir -p /workdir"} + for _, key := range keys { + parts = append(parts, "export "+key+"="+shellQuote(envs[key])) + } + parts = append(parts, "guidellm run --output kind=json,path=/workdir/dataset-preflight.json --output kind=csv,path=/workdir/dataset-preflight.csv > /workdir/dataset-preflight.log 2>&1") + return []string{"sh", "-lc", strings.Join(parts, " && ")}, nil +} + +func shellQuote(value string) string { + return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'" +} + +func BuildRunnerPodInput(input RunnerPodInput) *computeapi.ServerCreateInput { + root := int64(0) + ret := &computeapi.ServerCreateInput{ + ServerConfigs: computeapi.NewServerConfigs(), + VcpuCount: input.CPU, + VmemSize: input.MemoryMB, + AutoStart: true, + Pod: &computeapi.PodCreateInput{ + Containers: []*computeapi.PodContainerCreateInput{ + { + Name: "guidellm", + ContainerSpec: computeapi.ContainerSpec{ + ContainerSpec: commonapi.ContainerSpec{ + Image: input.Image, + Command: []string{"sh", "-lc", "while true; do sleep 3600; done"}, + AlwaysRestart: false, + SecurityContext: &commonapi.ContainerSecurityContext{ + RunAsUser: &root, + RunAsGroup: &root, + }, + }, + }, + }, + }, + }, + } + if input.PackageImageId != "" { + if input.PackageSizeMB <= 0 { + input.PackageSizeMB = 1024 + } + if input.PackageMountBase == "" { + input.PackageMountBase = "/data/benchmark-packages" + } + diskIndex := len(ret.Disks) + ret.Disks = append(ret.Disks, &computeapi.DiskConfig{ + DiskType: "data", + Format: "raw", + Fs: "ext4", + SizeMb: input.PackageSizeMB, + Index: diskIndex, + }) + ret.Pod.Containers[0].VolumeMounts = append(ret.Pod.Containers[0].VolumeMounts, &commonapi.ContainerVolumeMount{ + Type: commonapi.CONTAINER_VOLUME_MOUNT_TYPE_DISK, + MountPath: input.PackageMountBase, + Disk: &commonapi.ContainerVolumeMountDisk{ + Index: &diskIndex, + SubDirectory: strings.TrimPrefix(input.PackageMountBase, "/"), + PostOverlay: []*commonapi.ContainerVolumeMountDiskPostOverlay{ + { + Image: &commonapi.ContainerVolumeMountDiskPostImageOverlay{ + Id: input.PackageImageId, + }, + }, + }, + }, + Propagation: commonapi.MOUNTPROPAGATION_PROPAGATION_HOST_TO_CONTAINER, + }) + } + if input.ModelImageId != "" { + if input.ModelSizeMB <= 0 { + input.ModelSizeMB = 1024 + } + if input.ModelMountBase == "" { + input.ModelMountBase = "/data/models" + } + diskIndex := len(ret.Disks) + ret.Disks = append(ret.Disks, &computeapi.DiskConfig{ + DiskType: "data", + Format: "raw", + Fs: "ext4", + SizeMb: input.ModelSizeMB, + Index: diskIndex, + }) + ret.Pod.Containers[0].VolumeMounts = append(ret.Pod.Containers[0].VolumeMounts, &commonapi.ContainerVolumeMount{ + Type: commonapi.CONTAINER_VOLUME_MOUNT_TYPE_DISK, + MountPath: input.ModelMountBase, + ReadOnly: true, + Disk: &commonapi.ContainerVolumeMountDisk{ + Index: &diskIndex, + SubDirectory: input.ModelMountSubdirectory, + PostOverlay: []*commonapi.ContainerVolumeMountDiskPostOverlay{ + { + Image: &commonapi.ContainerVolumeMountDiskPostImageOverlay{ + Id: input.ModelImageId, + }, + }, + }, + }, + Propagation: commonapi.MOUNTPROPAGATION_PROPAGATION_HOST_TO_CONTAINER, + }) + } + ret.Hypervisor = computeapi.HYPERVISOR_POD + ret.Name = input.Name + ret.PreferHost = input.HostId + ret.Count = 1 + ret.Networks = []*computeapi.NetworkConfig{ + { + Network: input.NetworkId, + }, + } + return ret +} + +func GuideLLMEnvs(spec GuideLLMSpec) (map[string]string, error) { + backend, err := json.Marshal(spec.Backend) + if err != nil { + return nil, err + } + profile, err := json.Marshal(spec.Profile) + if err != nil { + return nil, err + } + tokenizer, err := json.Marshal(spec.Tokenizer) + if err != nil { + return nil, err + } + constraints, err := json.Marshal(spec.Constraints) + if err != nil { + return nil, err + } + data, err := json.Marshal(spec.Data) + if err != nil { + return nil, err + } + return map[string]string{ + "GUIDELLM__SPEC__BACKEND": string(backend), + "GUIDELLM__SPEC__PROFILE": string(profile), + "GUIDELLM__SPEC__TOKENIZER": string(tokenizer), + "GUIDELLM__SPEC__CONSTRAINTS": string(constraints), + "GUIDELLM__SPEC__DATA": string(data), + }, nil +} + +func tokenizerModel(model string) string { + if i := strings.LastIndex(model, ":"); i > 0 { + return model[:i] + } + return model +} + +func ParseMetricsCSV(path string) (*LLMBenchmarkMetrics, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + rows, err := csv.NewReader(f).ReadAll() + if err != nil { + return nil, err + } + if len(rows) >= 4 && len(rows[1]) > 0 && rows[1][0] == "Run ID" { + return parseGuideLLMv07MetricsCSV(rows), nil + } + if len(rows) < 2 { + return nil, fmt.Errorf("metrics csv has %d rows", len(rows)) + } + values := map[string]string{} + for i, key := range rows[0] { + if i < len(rows[1]) { + values[key] = rows[1][i] + } + } + ret := &LLMBenchmarkMetrics{ + RequestsPerSecondMean: parseFloatPtr(values["requests_per_second_mean"]), + RequestLatencyMeanSec: parseFloatPtr(values["request_latency_mean_sec"]), + RequestTotal: parseInt(values["request_total"]), + RequestSuccessful: parseInt(values["request_successful"]), + RequestErrored: parseInt(values["request_errored"]), + ErrorRate: parseFloatPtr(values["error_rate"]), + } + return ret, nil +} + +func parseGuideLLMv07MetricsCSV(rows [][]string) *LLMBenchmarkMetrics { + value := func(group, name, unit string) string { + for i := range rows[1] { + if i < len(rows[0]) && i < len(rows[2]) && i < len(rows[3]) && + rows[0][i] == group && rows[1][i] == name && (unit == "" || rows[2][i] == unit) { + return rows[3][i] + } + } + return "" + } + ret := &LLMBenchmarkMetrics{ + RequestsPerSecondMean: parseFloatPtr(value("Server Throughput", "Successful Requests/Sec", "Mean")), + RequestLatencyMeanSec: parseFloatPtr(value("Scheduler Metrics", "Request Time Avg", "Sec")), + RequestTotal: parseInt(value("Request Counts", "Total", "")), + RequestSuccessful: parseInt(value("Request Counts", "Successful", "")), + RequestErrored: parseInt(value("Request Counts", "Errored", "")), + } + if ret.RequestsPerSecondMean == nil { + if duration, err := strconv.ParseFloat(value("Timings", "Duration", "Sec"), 64); err == nil && duration > 0 { + v := float64(ret.RequestSuccessful) / duration + ret.RequestsPerSecondMean = &v + } + } + if ret.RequestTotal > 0 { + v := float64(ret.RequestErrored) / float64(ret.RequestTotal) + ret.ErrorRate = &v + } + return ret +} + +func parseFloatPtr(s string) *float64 { + if s == "" { + return nil + } + v, err := strconv.ParseFloat(s, 64) + if err != nil { + return nil + } + return &v +} + +func parseInt(s string) int { + v, _ := strconv.Atoi(s) + return v +} diff --git a/pkg/llm/models/llm_benchmark.go b/pkg/llm/models/llm_benchmark.go new file mode 100644 index 0000000000..7ef4d02bdd --- /dev/null +++ b/pkg/llm/models/llm_benchmark.go @@ -0,0 +1,1418 @@ +package models + +import ( + "context" + "database/sql" + stderrors "errors" + "os" + "path/filepath" + "strings" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/httputils" + "yunion.io/x/pkg/utils" + "yunion.io/x/sqlchemy" + + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + imageapi "yunion.io/x/onecloud/pkg/apis/image" + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/httperrors" + bench "yunion.io/x/onecloud/pkg/llm/benchmark" + "yunion.io/x/onecloud/pkg/llm/options" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/mcclient/modules/compute" + "yunion.io/x/onecloud/pkg/util/logclient" + "yunion.io/x/onecloud/pkg/util/stringutils2" +) + +func init() { + GetLLMBenchmarkManager() +} + +var llmBenchmarkManager *SLLMBenchmarkManager + +func GetLLMBenchmarkManager() *SLLMBenchmarkManager { + if llmBenchmarkManager != nil { + return llmBenchmarkManager + } + llmBenchmarkManager = &SLLMBenchmarkManager{ + SVirtualResourceBaseManager: db.NewVirtualResourceBaseManager( + SLLMBenchmark{}, + "llm_benchmarks_tbl", + "llm_benchmark", + "llm_benchmarks", + ), + } + llmBenchmarkManager.SetVirtualObject(llmBenchmarkManager) + return llmBenchmarkManager +} + +type SLLMBenchmarkManager struct { + db.SVirtualResourceBaseManager +} + +type LLMBenchmarkTokenizerMount struct { + ImageId string + ModelPath string + SizeMB int + MountBase string + MountSubdirectory string +} + +func selectBenchmarkTokenizerModel(llmType, model string, mounted []*SInstantModel) (*SInstantModel, error) { + matched := make([]*SInstantModel, 0, 1) + for _, candidate := range mounted { + if upstreamModelKeyFromInstantModel(llmType, candidate) == strings.TrimSpace(model) { + matched = append(matched, candidate) + } + } + if len(matched) == 1 { + return matched[0], nil + } + if len(matched) > 1 { + return nil, httperrors.NewInputParameterError("multiple mounted models match %s", model) + } + if len(mounted) == 1 { + return mounted[0], nil + } + return nil, httperrors.NewInputParameterError("cannot uniquely resolve tokenizer model %s", model) +} + +func buildBenchmarkTokenizerMount(llmType string, model *SInstantModel) (*LLMBenchmarkTokenizerMount, error) { + if model == nil || model.ImageId == "" || model.Status != imageapi.IMAGE_STATUS_ACTIVE || model.GetActualSizeMb() <= 0 { + return nil, httperrors.NewInvalidStatusError("benchmark tokenizer model image is not active") + } + subdirectory := "" + switch strings.ToLower(strings.TrimSpace(llmType)) { + case string(api.LLM_CONTAINER_VLLM): + subdirectory = api.LLM_VLLM + case string(api.LLM_CONTAINER_SGLANG): + subdirectory = api.LLM_SGLANG + default: + return nil, httperrors.NewInputParameterError("offline synthetic tokenizer is unsupported for %s", llmType) + } + modelPath := "" + for _, mount := range model.Mounts { + clean := filepath.Clean(mount) + rel, err := filepath.Rel(api.LLM_VLLM_BASE_PATH, clean) + if err == nil && rel != "." && rel != ".." && !filepath.IsAbs(rel) && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + modelPath = clean + break + } + } + if modelPath == "" { + return nil, httperrors.NewInvalidStatusError("benchmark tokenizer model has no mount below %s", api.LLM_VLLM_BASE_PATH) + } + return &LLMBenchmarkTokenizerMount{ + ImageId: model.ImageId, + ModelPath: modelPath, + SizeMB: int(model.GetActualSizeMb()) + 512, + MountBase: api.LLM_VLLM_BASE_PATH, + MountSubdirectory: subdirectory, + }, nil +} + +func resolveBenchmarkTokenizerMount(llm *SLLM, model string) (*LLMBenchmarkTokenizerMount, error) { + boolTrue := true + relations, err := llm.FetchModels(nil, &boolTrue, nil) + if err != nil { + return nil, errors.Wrap(err, "fetch mounted models") + } + mounted := make([]*SInstantModel, 0, len(relations)) + for i := range relations { + instant, err := GetInstantModelManager().GetInstantModelById(relations[i].InstantModelId) + if err != nil { + return nil, errors.Wrap(err, "fetch mounted instant model") + } + mounted = append(mounted, instant) + } + llmType := string(llm.GetLLMContainerDriver().GetType()) + selected, err := selectBenchmarkTokenizerModel(llmType, model, mounted) + if err != nil { + return nil, err + } + return buildBenchmarkTokenizerMount(llmType, selected) +} + +type SLLMBenchmark struct { + db.SVirtualResourceBase + + LLMId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required" index:"true"` + LLMDeploymentId string `width:"128" charset:"ascii" nullable:"true" list:"user" create:"optional" index:"true"` + LLMSkuId string `width:"128" charset:"ascii" nullable:"true" list:"user" create:"optional" index:"true"` + LLMImageId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"optional" index:"true"` + BenchmarkPackageId string `width:"128" charset:"ascii" nullable:"true" list:"user" create:"optional" index:"true"` + + Backend string `width:"64" charset:"ascii" nullable:"true" list:"user" create:"optional"` + Model string `width:"256" charset:"utf8" nullable:"true" list:"user" create:"optional" update:"user"` + TargetUrl string `charset:"utf8" length:"medium" nullable:"true" list:"user" create:"optional"` + RequestFormat string `width:"64" charset:"ascii" nullable:"true" list:"user" create:"optional"` + + Profile string `width:"32" charset:"ascii" nullable:"false" default:"constant" list:"user" create:"optional"` + DatasetName string `width:"64" charset:"ascii" nullable:"false" default:"synthetic_text" list:"user" create:"optional"` + DatasetInputTokens int `nullable:"true" list:"user" create:"optional" update:"user"` + DatasetOutputTokens int `nullable:"true" list:"user" create:"optional" update:"user"` + DatasetPath string `charset:"utf8" length:"medium" nullable:"true" list:"user" create:"optional"` + + RequestRate int `nullable:"true" list:"user" create:"optional" update:"user"` + TotalRequests int `nullable:"true" list:"user" create:"optional" update:"user"` + MaxDurationSeconds int `nullable:"true" list:"user" create:"optional" update:"user"` + MaxErrors int `nullable:"true" list:"user" create:"optional" update:"user"` + + State string `width:"32" charset:"ascii" nullable:"false" default:"pending" list:"user" create:"optional" update:"user" index:"true"` + StateMessage string `charset:"utf8" length:"medium" nullable:"true" list:"user" create:"optional" update:"user"` + TaskId string `width:"128" charset:"ascii" nullable:"true" list:"user" update:"user"` + StopRequested bool `nullable:"false" default:"false" list:"user" update:"user"` + + RunnerServerId string `width:"128" charset:"ascii" nullable:"true" list:"user" update:"user"` + RunnerContainerId string `width:"128" charset:"ascii" nullable:"true" list:"user" update:"user"` + + WorkDir string `charset:"utf8" length:"medium" nullable:"true" list:"user" create:"optional"` + LogPath string `charset:"utf8" length:"medium" nullable:"true" list:"user" update:"user"` + ResultJson string `charset:"utf8" length:"medium" nullable:"true" list:"user" update:"user"` + ResultCsv string `charset:"utf8" length:"medium" nullable:"true" list:"user" update:"user"` + + DatasetPreflight *api.LLMBenchmarkDatasetPreflight `json:"dataset_preflight,omitempty" length:"long" nullable:"true" list:"user" update:"user"` + PreflightLogPath string `charset:"utf8" length:"medium" nullable:"true" update:"user"` + PreflightResultJson string `charset:"utf8" length:"medium" nullable:"true" update:"user"` + RawPreflightLog string `charset:"utf8" length:"long" nullable:"true" update:"user"` + RawPreflightResult string `charset:"utf8" length:"long" nullable:"true" update:"user"` + + DatasetEvaluation *api.LLMBenchmarkDatasetEvaluation `json:"dataset_evaluation,omitempty" length:"long" nullable:"true" list:"user" update:"user"` + EvaluationResultJson string `charset:"utf8" length:"medium" nullable:"true" update:"user"` + EvaluationResultCsv string `charset:"utf8" length:"medium" nullable:"true" update:"user"` + EvaluationLogPath string `charset:"utf8" length:"medium" nullable:"true" update:"user"` + + ArtifactStorage string `width:"32" charset:"ascii" nullable:"true" list:"user" update:"user"` + ArtifactStorageMessage string `charset:"utf8" length:"medium" nullable:"true" list:"user" update:"user"` + + TargetSnapshot string `charset:"utf8" length:"long" nullable:"true" list:"user" create:"optional" update:"user"` + GuideLLMSpec string `charset:"utf8" length:"long" nullable:"true" list:"user" create:"optional" update:"user"` + RawMetrics string `charset:"utf8" length:"long" nullable:"true" list:"user" update:"user"` + RawLog string `charset:"utf8" length:"long" nullable:"true" update:"user"` + RawCsv string `charset:"utf8" length:"long" nullable:"true" update:"user"` + + RequestsPerSecondMean float64 `nullable:"true" list:"user" update:"user"` + RequestLatencyMeanSec float64 `nullable:"true" list:"user" update:"user"` + + RequestTotal int `nullable:"true" list:"user" update:"user"` + RequestSuccessful int `nullable:"true" list:"user" update:"user"` + RequestErrored int `nullable:"true" list:"user" update:"user"` + ErrorRate float64 `nullable:"true" list:"user" update:"user"` +} + +func (man *SLLMBenchmarkManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input *api.LLMBenchmarkCreateInput) (*api.LLMBenchmarkCreateInput, error) { + var err error + input.VirtualResourceCreateInput, err = man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.VirtualResourceCreateInput) + if err != nil { + return input, errors.Wrap(err, "validate VirtualResourceCreateInput") + } + return prepareLLMBenchmarkCreateInput(ctx, userCred, ownerId, input) +} + +func prepareLLMBenchmarkCreateInput(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, input *api.LLMBenchmarkCreateInput) (*api.LLMBenchmarkCreateInput, error) { + llm, dep, err := resolveBenchmarkTarget(ctx, userCred, input) + if err != nil { + return input, err + } + sku, err := llm.GetLLMSku(llm.LLMSkuId) + if err != nil { + return input, errors.Wrap(err, "GetLLMSku") + } + image, err := resolveBenchmarkImage(ctx, userCred, input.BenchmarkImage) + if err != nil { + return input, err + } + pkg, err := resolveBenchmarkPackage(ctx, userCred, input.BenchmarkPackage) + if err != nil { + return input, err + } + if pkg != nil { + benchmarkProjectID := "" + if ownerId != nil { + benchmarkProjectID = ownerId.GetProjectId() + } + if err := validateBenchmarkPackageProject(pkg.ProjectId, benchmarkProjectID); err != nil { + return input, err + } + } + access, err := llm.GetLLMAccessUrlInfo(ctx, userCred, jsonutils.NewDict()) + if err != nil { + return input, errors.Wrap(err, "GetLLMAccessUrlInfo") + } + targetURL := "" + if access != nil { + targetURL = access.InternalUrl + if targetURL == "" { + targetURL = access.LoginUrl + } + } + if targetURL == "" { + return input, errors.Wrap(httperrors.ErrInvalidStatus, "target url is empty") + } + + defaultLLMBenchmarkInput(input, pkg) + if err := validateLLMBenchmarkInput(input, pkg != nil); err != nil { + return input, err + } + if input.Model == "" { + input.Model = resolveBenchmarkModel(ctx, targetURL, sku.LLMType, llm) + } + if pkg == nil { + if _, err := resolveBenchmarkTokenizerMount(llm, input.Model); err != nil { + return input, errors.Wrap(err, "resolve offline benchmark tokenizer") + } + } + + input.LLMId = llm.Id + input.LLMSkuId = sku.Id + input.LLMImageId = image.Id + input.BenchmarkPackageId = "" + if pkg != nil { + input.BenchmarkPackageId = pkg.Id + } + input.LLMDeploymentId = llm.LLMDeploymentId + if dep != nil { + input.LLMDeploymentId = dep.Id + } + input.Backend = sku.LLMType + input.TargetUrl = targetURL + workDir := benchmarkWorkDirRoot() + input.WorkDir = filepath.Join(workDir, input.Name) + if !benchmarkWorkDirIsSafe(workDir, input.WorkDir) { + return input, httperrors.NewInputParameterError("name produces unsafe benchmark workdir") + } + input.TargetSnapshot = jsonutils.Marshal(map[string]interface{}{ + "llm_id": llm.Id, + "llm_name": llm.Name, + "llm_status": llm.Status, + "llm_deployment_id": input.LLMDeploymentId, + "llm_sku_id": sku.Id, + "llm_sku_name": sku.Name, + "backend": sku.LLMType, + "llm_image_id": image.Id, + "llm_image_name": image.Name, + "benchmark_package_id": input.BenchmarkPackageId, + "target_url": targetURL, + "request_format": input.RequestFormat, + "model": input.Model, + "request_rate": input.RequestRate, + "total_requests": input.TotalRequests, + }).String() + spec := bench.BuildGuideLLMSpec(bench.GuideLLMSpecInput{ + TargetURL: input.TargetUrl, + RequestFormat: input.RequestFormat, + Model: input.Model, + RequestRate: input.RequestRate, + TotalRequests: input.TotalRequests, + MaxDurationSeconds: input.MaxDurationSeconds, + MaxErrors: input.MaxErrors, + DatasetInputTokens: input.DatasetInputTokens, + DatasetOutputTokens: input.DatasetOutputTokens, + DatasetPath: input.DatasetPath, + }) + input.GuideLLMSpec = jsonutils.Marshal(spec).String() + return input, nil +} + +func benchmarkWorkDirRoot() string { + if root := options.Options.LLMBenchmarkWorkDir; root != "" { + return root + } + return "/opt/cloud/workspace/llm/benchmarks" +} + +func benchmarkWorkDirIsSafe(root, path string) bool { + rel, err := filepath.Rel(filepath.Clean(root), filepath.Clean(path)) + return err == nil && rel != "." && rel != ".." && !filepath.IsAbs(rel) && + !strings.HasPrefix(rel, ".."+string(filepath.Separator)) +} + +func resolveBenchmarkTarget(ctx context.Context, userCred mcclient.TokenCredential, input *api.LLMBenchmarkCreateInput) (*SLLM, *SLLMDeployment, error) { + if input.LLMId != "" && input.LLMDeploymentId != "" { + return nil, nil, errors.Wrap(httperrors.ErrInputParameter, "llm_id and llm_deployment_id are mutually exclusive") + } + if input.LLMDeploymentId != "" { + depObj, err := GetLLMDeploymentManager().FetchByIdOrName(ctx, userCred, input.LLMDeploymentId) + if err != nil { + return nil, nil, errors.Wrap(err, "fetch LLMDeployment") + } + dep := depObj.(*SLLMDeployment) + llm := &SLLM{} + err = GetLLMManager().Query(). + Equals("llm_deployment_id", dep.Id). + Equals("status", api.LLM_STATUS_RUNNING). + Asc("created_at"). + First(llm) + if err != nil { + if errors.Cause(err) == sql.ErrNoRows { + return nil, nil, errors.Wrapf(httperrors.ErrInvalidStatus, "deployment %s has no running llm", dep.Name) + } + return nil, nil, errors.Wrap(err, "query running deployment LLM") + } + llm.SetModelManager(GetLLMManager(), llm) + return llm, dep, nil + } + if input.LLMId == "" { + return nil, nil, errors.Wrap(httperrors.ErrMissingParameter, "llm_id or llm_deployment_id") + } + llmObj, err := GetLLMManager().FetchByIdOrName(ctx, userCred, input.LLMId) + if err != nil { + return nil, nil, errors.Wrap(err, "fetch LLM") + } + llm := llmObj.(*SLLM) + if llm.Status != api.LLM_STATUS_RUNNING { + return nil, nil, errors.Wrapf(httperrors.ErrInvalidStatus, "llm %s status is %s", llm.Name, llm.Status) + } + return llm, nil, nil +} + +func resolveBenchmarkModel(ctx context.Context, targetURL string, llmType string, llm *SLLM) string { + if providerType, ok := benchmarkProviderType(llmType); ok { + out, err := GetLLMManager().performProviderModels(ctx, api.LLMProviderModelsInput{ + URL: targetURL, + ProviderType: providerType, + }) + if err == nil { + for _, model := range out.Models { + if model = strings.TrimSpace(model); model != "" { + return model + } + } + } + } + if infos, err := llm.FetchMountedModelInfo(); err == nil && len(infos) > 0 { + return upstreamModelKeyFromMountedInfo(llmType, &infos[0]) + } + return "" +} + +func benchmarkProviderType(llmType string) (api.LLMClientType, bool) { + switch strings.ToLower(strings.TrimSpace(llmType)) { + case string(api.LLM_CONTAINER_OLLAMA): + return api.LLM_CLIENT_OLLAMA, true + case string(api.LLM_CONTAINER_VLLM), string(api.LLM_CONTAINER_SGLANG): + return api.LLM_CLIENT_OPENAI, true + default: + return "", false + } +} + +func defaultLLMBenchmarkInput(input *api.LLMBenchmarkCreateInput, pkg *SLLMBenchmarkPackage) { + if input.RequestFormat == "" { + input.RequestFormat = api.LLMBenchmarkDefaultRequestFormat + } + if input.Profile == "" { + input.Profile = api.LLMBenchmarkProfileConstant + } + if pkg != nil { + if input.DatasetName == "" || input.DatasetName == api.LLMBenchmarkDatasetSyntheticText { + input.DatasetName = api.LLMBenchmarkDatasetPackage + } + if input.DatasetPath == "" { + input.DatasetPath = pkg.DatasetPath + } + } else if input.DatasetName == "" { + input.DatasetName = api.LLMBenchmarkDatasetSyntheticText + } + if input.RequestRate <= 0 { + input.RequestRate = options.Options.LLMBenchmarkDefaultRequestRate + if input.RequestRate <= 0 { + input.RequestRate = 1 + } + } + if input.TotalRequests <= 0 { + input.TotalRequests = options.Options.LLMBenchmarkDefaultTotalRequests + if input.TotalRequests <= 0 { + input.TotalRequests = 100 + } + } + if input.MaxDurationSeconds <= 0 { + input.MaxDurationSeconds = 600 + } + if input.MaxErrors <= 0 { + input.MaxErrors = 10 + } + if input.DatasetInputTokens <= 0 { + input.DatasetInputTokens = options.Options.LLMBenchmarkDefaultInputTokens + if input.DatasetInputTokens <= 0 { + input.DatasetInputTokens = 1024 + } + } + if input.DatasetOutputTokens <= 0 { + input.DatasetOutputTokens = options.Options.LLMBenchmarkDefaultOutputTokens + if input.DatasetOutputTokens <= 0 { + input.DatasetOutputTokens = 128 + } + } +} + +func validateLLMBenchmarkInput(input *api.LLMBenchmarkCreateInput, hasPackage bool) error { + if input.RequestFormat != api.LLMBenchmarkDefaultRequestFormat { + return errors.Wrap(httperrors.ErrInputParameter, "request_format only supports "+api.LLMBenchmarkDefaultRequestFormat) + } + if input.Profile != api.LLMBenchmarkProfileConstant { + return errors.Wrap(httperrors.ErrInputParameter, "profile only supports constant") + } + if hasPackage { + if input.DatasetName != api.LLMBenchmarkDatasetPackage { + return errors.Wrap(httperrors.ErrInputParameter, "dataset_name must be benchmark_package when benchmark_package is set") + } + if strings.TrimSpace(input.DatasetPath) == "" { + return httperrors.NewMissingParameterError("dataset_path") + } + } else if input.DatasetName != api.LLMBenchmarkDatasetSyntheticText { + return errors.Wrap(httperrors.ErrInputParameter, "dataset_name only supports synthetic_text") + } + if max := options.Options.LLMBenchmarkMaxRequestRate; max > 0 && input.RequestRate > max { + return errors.Wrapf(httperrors.ErrInputParameter, "request_rate must be <= %d", max) + } + if max := options.Options.LLMBenchmarkMaxTotalRequests; max > 0 && input.TotalRequests > max { + return errors.Wrapf(httperrors.ErrInputParameter, "total_requests must be <= %d", max) + } + if max := options.Options.LLMBenchmarkMaxDurationSeconds; max > 0 && input.MaxDurationSeconds > max { + return errors.Wrapf(httperrors.ErrInputParameter, "max_duration_seconds must be <= %d", max) + } + return nil +} + +func validateBenchmarkMutableState(state string) error { + if utils.IsInStringArray(state, []string{ + api.LLMBenchmarkStateCompleted, + api.LLMBenchmarkStateStopped, + api.LLMBenchmarkStateError, + }) { + return nil + } + return httperrors.NewInvalidStatusError("benchmark is %s", state) +} + +func (b *SLLMBenchmark) ValidateUpdateCondition(ctx context.Context) error { + return validateBenchmarkMutableState(b.State) +} + +func positiveBenchmarkUpdate(name string, value *int) error { + if value != nil && *value <= 0 { + return httperrors.NewInputParameterError("%s must be greater than 0", name) + } + return nil +} + +func (b *SLLMBenchmark) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.LLMBenchmarkUpdateInput) (api.LLMBenchmarkUpdateInput, error) { + var err error + input.VirtualResourceBaseUpdateInput, err = b.SVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, input.VirtualResourceBaseUpdateInput) + if err != nil { + return input, errors.Wrap(err, "ValidateUpdateData") + } + if err := validateBenchmarkMutableState(b.State); err != nil { + return input, err + } + for name, value := range map[string]*int{ + "request_rate": input.RequestRate, + "total_requests": input.TotalRequests, + "max_duration_seconds": input.MaxDurationSeconds, + "max_errors": input.MaxErrors, + "dataset_input_tokens": input.DatasetInputTokens, + "dataset_output_tokens": input.DatasetOutputTokens, + } { + if err := positiveBenchmarkUpdate(name, value); err != nil { + return input, err + } + } + if b.BenchmarkPackageId != "" && (input.DatasetInputTokens != nil || input.DatasetOutputTokens != nil) { + return input, httperrors.NewInputParameterError("dataset token fields only apply to synthetic_text") + } + candidate := benchmarkCreateInputFromModel(b) + applyLLMBenchmarkUpdateInput(candidate, input) + if err := validateLLMBenchmarkInput(candidate, b.BenchmarkPackageId != ""); err != nil { + return input, err + } + if b.BenchmarkPackageId == "" && benchmarkUpdateChangesRunConfig(input) { + obj, err := GetLLMManager().FetchById(b.LLMId) + if err != nil { + return input, errors.Wrap(err, "fetch benchmark LLM") + } + if _, err := resolveBenchmarkTokenizerMount(obj.(*SLLM), candidate.Model); err != nil { + return input, errors.Wrap(err, "resolve offline benchmark tokenizer") + } + } + return input, nil +} + +func benchmarkCreateInputFromModel(b *SLLMBenchmark) *api.LLMBenchmarkCreateInput { + ret := &api.LLMBenchmarkCreateInput{ + LLMId: b.LLMId, + LLMDeploymentId: b.LLMDeploymentId, + LLMImageId: b.LLMImageId, + BenchmarkPackageId: b.BenchmarkPackageId, + RequestFormat: b.RequestFormat, + Model: b.Model, + Profile: b.Profile, + RequestRate: b.RequestRate, + TotalRequests: b.TotalRequests, + MaxDurationSeconds: b.MaxDurationSeconds, + MaxErrors: b.MaxErrors, + DatasetName: b.DatasetName, + DatasetInputTokens: b.DatasetInputTokens, + DatasetOutputTokens: b.DatasetOutputTokens, + DatasetPath: b.DatasetPath, + Backend: b.Backend, + TargetUrl: b.TargetUrl, + WorkDir: b.WorkDir, + TargetSnapshot: b.TargetSnapshot, + GuideLLMSpec: b.GuideLLMSpec, + } + ret.Name = b.Name + ret.Description = b.Description + return ret +} + +func applyLLMBenchmarkUpdateInput(candidate *api.LLMBenchmarkCreateInput, input api.LLMBenchmarkUpdateInput) { + if input.Model != nil { + candidate.Model = strings.TrimSpace(*input.Model) + } + if input.RequestRate != nil { + candidate.RequestRate = *input.RequestRate + } + if input.TotalRequests != nil { + candidate.TotalRequests = *input.TotalRequests + } + if input.MaxDurationSeconds != nil { + candidate.MaxDurationSeconds = *input.MaxDurationSeconds + } + if input.MaxErrors != nil { + candidate.MaxErrors = *input.MaxErrors + } + if input.DatasetInputTokens != nil { + candidate.DatasetInputTokens = *input.DatasetInputTokens + } + if input.DatasetOutputTokens != nil { + candidate.DatasetOutputTokens = *input.DatasetOutputTokens + } +} + +func benchmarkUpdateChangesRunConfig(input api.LLMBenchmarkUpdateInput) bool { + return input.Model != nil || + input.RequestRate != nil || + input.TotalRequests != nil || + input.MaxDurationSeconds != nil || + input.MaxErrors != nil || + input.DatasetInputTokens != nil || + input.DatasetOutputTokens != nil +} + +func buildLLMBenchmarkUpdateData(b *SLLMBenchmark, input api.LLMBenchmarkUpdateInput) *jsonutils.JSONDict { + if !benchmarkUpdateChangesRunConfig(input) { + return jsonutils.NewDict() + } + candidate := benchmarkCreateInputFromModel(b) + applyLLMBenchmarkUpdateInput(candidate, input) + + spec := bench.BuildGuideLLMSpec(bench.GuideLLMSpecInput{ + TargetURL: candidate.TargetUrl, + RequestFormat: candidate.RequestFormat, + Model: candidate.Model, + RequestRate: candidate.RequestRate, + TotalRequests: candidate.TotalRequests, + MaxDurationSeconds: candidate.MaxDurationSeconds, + MaxErrors: candidate.MaxErrors, + DatasetInputTokens: candidate.DatasetInputTokens, + DatasetOutputTokens: candidate.DatasetOutputTokens, + DatasetPath: candidate.DatasetPath, + }) + snapshot, _ := jsonutils.ParseString(b.TargetSnapshot) + snapshotDict, ok := snapshot.(*jsonutils.JSONDict) + if !ok { + snapshotDict = jsonutils.NewDict() + } + snapshotDict.Set("model", jsonutils.NewString(candidate.Model)) + snapshotDict.Set("request_rate", jsonutils.NewInt(int64(candidate.RequestRate))) + snapshotDict.Set("total_requests", jsonutils.NewInt(int64(candidate.TotalRequests))) + + data := jsonutils.NewDict() + if input.Model != nil { + data.Set("model", jsonutils.NewString(candidate.Model)) + } + if input.RequestRate != nil { + data.Set("request_rate", jsonutils.NewInt(int64(candidate.RequestRate))) + } + if input.TotalRequests != nil { + data.Set("total_requests", jsonutils.NewInt(int64(candidate.TotalRequests))) + } + if input.MaxDurationSeconds != nil { + data.Set("max_duration_seconds", jsonutils.NewInt(int64(candidate.MaxDurationSeconds))) + } + if input.MaxErrors != nil { + data.Set("max_errors", jsonutils.NewInt(int64(candidate.MaxErrors))) + } + if input.DatasetInputTokens != nil { + data.Set("dataset_input_tokens", jsonutils.NewInt(int64(candidate.DatasetInputTokens))) + } + if input.DatasetOutputTokens != nil { + data.Set("dataset_output_tokens", jsonutils.NewInt(int64(candidate.DatasetOutputTokens))) + } + data.Set("guide_llm_spec", jsonutils.NewString(jsonutils.Marshal(spec).String())) + data.Set("target_snapshot", jsonutils.NewString(snapshotDict.String())) + data.Set("state", jsonutils.NewString(api.LLMBenchmarkStateStopped)) + for _, field := range []string{ + "state_message", "task_id", "runner_server_id", "runner_container_id", + "log_path", "result_json", "result_csv", "raw_metrics", "raw_log", "raw_csv", + "preflight_log_path", "preflight_result_json", + "raw_preflight_log", "raw_preflight_result", + "evaluation_result_json", "evaluation_result_csv", "evaluation_log_path", + "artifact_storage", "artifact_storage_message", + } { + data.Set(field, jsonutils.NewString("")) + } + data.Set("dataset_preflight", jsonutils.JSONNull) + data.Set("dataset_evaluation", jsonutils.JSONNull) + data.Set("stop_requested", jsonutils.JSONFalse) + for _, field := range []string{ + "requests_per_second_mean", "request_latency_mean_sec", "error_rate", + } { + data.Set(field, jsonutils.NewFloat64(0)) + } + for _, field := range []string{ + "request_total", "request_successful", "request_errored", + } { + data.Set(field, jsonutils.NewInt(0)) + } + return data +} + +func sanitizeLLMBenchmarkUpdateData(data *jsonutils.JSONDict) { + for _, field := range []string{ + "state", "state_message", "task_id", "stop_requested", + "runner_server_id", "runner_container_id", + "log_path", "result_json", "result_csv", + "dataset_preflight", "preflight_log_path", "preflight_result_json", + "raw_preflight_log", "raw_preflight_result", + "dataset_evaluation", "evaluation_result_json", "evaluation_result_csv", "evaluation_log_path", + "artifact_storage", "artifact_storage_message", + "target_snapshot", "guide_llm_spec", "raw_metrics", "raw_log", "raw_csv", + "requests_per_second_mean", "request_latency_mean_sec", + "request_total", "request_successful", "request_errored", "error_rate", + } { + data.RemoveIgnoreCase(field) + } +} + +func (b *SLLMBenchmark) PreUpdate(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) { + dataDict := data.(*jsonutils.JSONDict) + input := api.LLMBenchmarkUpdateInput{} + _ = dataDict.Unmarshal(&input) + if benchmarkUpdateChangesRunConfig(input) { + if err := b.CleanupArtifacts(ctx); err != nil { + log.Warningf("cleanup benchmark %s artifacts before update: %s", b.Id, err) + } + } + sanitizeLLMBenchmarkUpdateData(dataDict) + dataDict.Update(buildLLMBenchmarkUpdateData(b, input)) + b.SVirtualResourceBase.PreUpdate(ctx, userCred, query, data) +} + +func resolveBenchmarkPackage(ctx context.Context, userCred mcclient.TokenCredential, packageName string) (*SLLMBenchmarkPackage, error) { + if packageName == "" { + return nil, nil + } + obj, err := GetLLMBenchmarkPackageManager().FetchByIdOrName(ctx, userCred, packageName) + if err != nil { + return nil, errors.Wrap(err, "fetch benchmark package") + } + pkg := obj.(*SLLMBenchmarkPackage) + if pkg.ImageId == "" { + return nil, httperrors.NewInvalidStatusError("benchmark package %s has no image", pkg.Name) + } + if pkg.Status != imageapi.IMAGE_STATUS_ACTIVE { + return nil, httperrors.NewInvalidStatusError("benchmark package %s is %s", pkg.Name, pkg.Status) + } + return pkg, nil +} + +func PrepareLLMBenchmarkCreateInput(pkg *SLLMBenchmarkPackage, spec *api.LLMBenchmarkCreateInput) (*api.LLMBenchmarkCreateInput, error) { + if spec == nil { + return nil, nil + } + if pkg == nil || pkg.Id == "" { + return nil, errors.Wrap(httperrors.ErrInputParameter, "empty benchmark package") + } + if pkg.Status != imageapi.IMAGE_STATUS_ACTIVE || pkg.ImageId == "" { + return nil, httperrors.NewInvalidStatusError("benchmark package %s is not active", pkg.Name) + } + + input := *spec + input.BenchmarkPackage = pkg.Id + input.BenchmarkPackageId = pkg.Id + input.DatasetName = api.LLMBenchmarkDatasetPackage + input.DatasetPath = pkg.DatasetPath + return &input, nil +} + +func validateBenchmarkPackageUnusedCount(count int) error { + if count > 0 { + return errors.Wrap(httperrors.ErrInvalidStatus, "benchmark package is still used by benchmarks") + } + return nil +} + +func validateBenchmarkPackageProject(packageProjectID, benchmarkProjectID string) error { + if packageProjectID != benchmarkProjectID { + return httperrors.NewInputParameterError("benchmark package and benchmark must belong to the same project") + } + return nil +} + +func CountBenchmarkPackageReferences(packageID, excludeBenchmarkID string) (int, error) { + if packageID == "" { + return 0, nil + } + q := GetLLMBenchmarkManager().Query(). + Equals("benchmark_package_id", packageID). + IsFalse("deleted") + if excludeBenchmarkID != "" { + q = q.NotEquals("id", excludeBenchmarkID) + } + count, err := q.CountWithError() + if err != nil { + return 0, errors.Wrap(err, "count benchmark package references") + } + return count, nil +} + +func ValidateBenchmarkPackageUnused(packageID, excludeBenchmarkID string) error { + count, err := CountBenchmarkPackageReferences(packageID, excludeBenchmarkID) + if err != nil { + return err + } + return validateBenchmarkPackageUnusedCount(count) +} + +func buildLLMBenchmarkCopyCreateInput(source *SLLMBenchmark, input api.LLMBenchmarkCopyInput) (*api.LLMBenchmarkCreateInput, error) { + if strings.TrimSpace(input.Name) == "" { + return nil, httperrors.NewMissingParameterError("name") + } + if strings.TrimSpace(input.LLMDeploymentId) == "" { + return nil, httperrors.NewMissingParameterError("llm_deployment_id") + } + if source.BenchmarkPackageId != "" && (input.DatasetInputTokens != nil || input.DatasetOutputTokens != nil) { + return nil, httperrors.NewInputParameterError("dataset token fields only apply to synthetic_text") + } + ret := &api.LLMBenchmarkCreateInput{ + LLMDeploymentId: input.LLMDeploymentId, + BenchmarkImage: source.LLMImageId, + BenchmarkPackage: source.BenchmarkPackageId, + RequestFormat: source.RequestFormat, + Profile: source.Profile, + RequestRate: source.RequestRate, + TotalRequests: source.TotalRequests, + MaxDurationSeconds: source.MaxDurationSeconds, + MaxErrors: source.MaxErrors, + DatasetName: source.DatasetName, + DatasetInputTokens: source.DatasetInputTokens, + DatasetOutputTokens: source.DatasetOutputTokens, + DatasetPath: source.DatasetPath, + } + ret.Name = input.Name + ret.Description = source.Description + if input.Description != nil { + ret.Description = *input.Description + } + if input.Model != nil { + ret.Model = strings.TrimSpace(*input.Model) + } + if input.RequestRate != nil { + ret.RequestRate = *input.RequestRate + } + if input.TotalRequests != nil { + ret.TotalRequests = *input.TotalRequests + } + if input.MaxDurationSeconds != nil { + ret.MaxDurationSeconds = *input.MaxDurationSeconds + } + if input.MaxErrors != nil { + ret.MaxErrors = *input.MaxErrors + } + if input.DatasetInputTokens != nil { + ret.DatasetInputTokens = *input.DatasetInputTokens + } + if input.DatasetOutputTokens != nil { + ret.DatasetOutputTokens = *input.DatasetOutputTokens + } + return ret, nil +} + +func resolveBenchmarkImage(ctx context.Context, userCred mcclient.TokenCredential, imageName string) (*SLLMImage, error) { + var image *SLLMImage + if imageName != "" { + obj, err := GetLLMImageManager().FetchByIdOrName(ctx, userCred, imageName) + if err != nil { + return nil, errors.Wrap(err, "fetch benchmark image") + } + image = obj.(*SLLMImage) + } else { + defaultImage := options.Options.LLMBenchmarkDefaultImage + if defaultImage == "" { + defaultImage = api.LLMBenchmarkDefaultImage + } + name, label := parseImageRef(defaultImage) + image = &SLLMImage{} + err := GetLLMImageManager().Query(). + Equals("image_name", name). + Equals("image_label", label). + Equals("llm_type", string(api.LLM_IMAGE_TYPE_BENCHMARK)). + First(image) + if err != nil { + if errors.Cause(err) == sql.ErrNoRows { + return nil, httperrors.NewNotFoundError("default benchmark llm_image %s not found, create it first", defaultImage) + } + return nil, errors.Wrap(err, "query default benchmark image") + } + image.SetModelManager(GetLLMImageManager(), image) + } + if image.LLMType != string(api.LLM_IMAGE_TYPE_BENCHMARK) { + return nil, errors.Wrapf(httperrors.ErrInputParameter, "image %s is not benchmark type", image.Name) + } + return image, nil +} + +func parseImageRef(ref string) (string, string) { + idx := strings.LastIndex(ref, ":") + if idx <= 0 { + return ref, "latest" + } + return ref[:idx], ref[idx+1:] +} + +func (man *SLLMBenchmarkManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, input api.LLMBenchmarkListInput) (*sqlchemy.SQuery, error) { + q, err := man.SVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, input.VirtualResourceListInput) + if err != nil { + return nil, errors.Wrap(err, "SVirtualResourceBaseManager.ListItemFilter") + } + if input.LLMId != "" { + q = q.Equals("llm_id", input.LLMId) + } + if input.LLMDeploymentId != "" { + depObj, err := GetLLMDeploymentManager().FetchByIdOrName(ctx, userCred, input.LLMDeploymentId) + if err != nil { + return nil, errors.Wrap(err, "fetch LLMDeployment") + } + q = q.Equals("llm_deployment_id", depObj.GetId()) + } + if input.State != "" { + q = q.Equals("state", input.State) + } + return q, nil +} + +func (man *SLLMBenchmarkManager) QueryDistinctExtraField(q *sqlchemy.SQuery, field string) (*sqlchemy.SQuery, error) { + q, err := man.SVirtualResourceBaseManager.QueryDistinctExtraField(q, field) + if err == nil { + return q, nil + } + if !isLLMBenchmarkDeploymentExtraField(field) { + return q, httperrors.ErrNotFound + } + depQ := GetLLMDeploymentManager().Query("id", "name").Distinct().SubQuery() + q.AppendField(depQ.Field("name", field)) + q = q.Join(depQ, sqlchemy.Equals(q.Field("llm_deployment_id"), depQ.Field("id"))) + q.GroupBy(depQ.Field("name")) + return q, nil +} + +func isLLMBenchmarkDeploymentExtraField(field string) bool { + return field == "llm_deployment" +} + +func (man *SLLMBenchmarkManager) FetchCustomizeColumns( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + objs []interface{}, + fields stringutils2.SSortedStrings, + isList bool, +) []api.LLMBenchmarkDetails { + virtRows := man.SVirtualResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) + benchmarks := make([]SLLMBenchmark, len(objs)) + jsonutils.Update(&benchmarks, objs) + + rows := make([]api.LLMBenchmarkDetails, len(objs)) + deploymentIds := make([]string, 0, len(objs)) + for i := range rows { + rows[i].VirtualResourceDetails = virtRows[i] + if benchmarks[i].LLMDeploymentId != "" { + deploymentIds = append(deploymentIds, benchmarks[i].LLMDeploymentId) + } + } + if len(deploymentIds) == 0 { + return rows + } + + deploymentNames, err := db.FetchIdNameMap2(GetLLMDeploymentManager(), deploymentIds) + if err != nil { + return rows + } + for i := range rows { + rows[i].LLMDeployment = deploymentNames[benchmarks[i].LLMDeploymentId] + } + return rows +} + +func (man *SLLMBenchmarkManager) CreateAndStart(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, input *api.LLMBenchmarkCreateInput) (*SLLMBenchmark, error) { + data := jsonutils.Marshal(input) + obj, err := db.DoCreate(man, ctx, userCred, nil, data, ownerId) + if err != nil { + return nil, errors.Wrap(err, "DoCreate benchmark") + } + benchmark := obj.(*SLLMBenchmark) + func() { + lockman.LockObject(ctx, benchmark) + defer lockman.ReleaseObject(ctx, benchmark) + benchmark.PostCreate(ctx, userCred, ownerId, nil, data) + if err := man.GetExtraHook().AfterPostCreate(ctx, userCred, ownerId, benchmark, nil, data); err != nil { + logclient.AddActionLogWithContext(ctx, benchmark, logclient.ACT_POST_CREATE_HOOK, err, userCred, false) + } + }() + notes := benchmark.GetShortDesc(ctx) + db.OpsLog.LogEvent(benchmark, db.ACT_CREATE, notes, userCred) + logclient.AddActionLogWithContext(ctx, benchmark, logclient.ACT_CREATE, notes, userCred, true) + man.OnCreateComplete(ctx, []db.IModel{benchmark}, userCred, ownerId, nil, []jsonutils.JSONObject{data}) + return benchmark, nil +} + +func (b *SLLMBenchmark) PerformCopy(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.LLMBenchmarkCopyInput) (jsonutils.JSONObject, error) { + createInput, err := buildLLMBenchmarkCopyCreateInput(b, input) + if err != nil { + return nil, err + } + benchmark, err := GetLLMBenchmarkManager().CreateAndStart(ctx, userCred, b.GetOwnerId(), createInput) + if err != nil { + return nil, err + } + ret := jsonutils.NewDict() + ret.Set("benchmark_id", jsonutils.NewString(benchmark.Id)) + ret.Set("state", jsonutils.NewString(benchmark.State)) + return ret, nil +} + +func (b *SLLMBenchmark) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) { + b.SVirtualResourceBase.PostCreate(ctx, userCred, ownerId, query, data) + if err := b.StartRunTask(ctx, userCred, ""); err != nil { + _ = b.SetState(ctx, userCred, api.LLMBenchmarkStateError, err.Error()) + } +} + +func (b *SLLMBenchmark) SetState(ctx context.Context, userCred mcclient.TokenCredential, state string, message string) error { + _, err := db.Update(b, func() error { + b.State = state + b.StateMessage = message + return nil + }) + return err +} + +func (b *SLLMBenchmark) FinishRun(ctx context.Context, runErr error) (string, error) { + lockman.LockObject(ctx, b) + defer lockman.ReleaseObject(ctx, b) + + obj, err := GetLLMBenchmarkManager().FetchById(b.Id) + if err != nil { + return "", err + } + current := obj.(*SLLMBenchmark) + state, message := benchmarkRunFinalState(current.State, current.StopRequested, runErr) + _, err = db.Update(current, func() error { + current.State = state + current.StateMessage = message + return nil + }) + if err == nil { + b.State = state + b.StateMessage = message + b.StopRequested = current.StopRequested + } + return state, err +} + +func (b *SLLMBenchmark) SetRunner(ctx context.Context, serverId string, containerId string) error { + _, err := db.Update(b, func() error { + b.RunnerServerId = serverId + b.RunnerContainerId = containerId + return nil + }) + return err +} + +func resetLLMBenchmarkResultFields(b *SLLMBenchmark) { + b.StateMessage = "" + b.TaskId = "" + b.StopRequested = false + b.RunnerServerId = "" + b.RunnerContainerId = "" + b.LogPath = "" + b.ResultJson = "" + b.ResultCsv = "" + b.RawMetrics = "" + b.RawLog = "" + b.RawCsv = "" + b.DatasetPreflight = nil + b.PreflightLogPath = "" + b.PreflightResultJson = "" + b.RawPreflightLog = "" + b.RawPreflightResult = "" + b.DatasetEvaluation = nil + b.EvaluationResultJson = "" + b.EvaluationResultCsv = "" + b.EvaluationLogPath = "" + b.ArtifactStorage = "" + b.ArtifactStorageMessage = "" + b.RequestsPerSecondMean = 0 + b.RequestLatencyMeanSec = 0 + b.RequestTotal = 0 + b.RequestSuccessful = 0 + b.RequestErrored = 0 + b.ErrorRate = 0 +} + +func (b *SLLMBenchmark) StartRunTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error { + task, err := taskman.TaskManager.NewTask(ctx, "LLMBenchmarkRunTask", b, userCred, nil, parentTaskId, "", nil) + if err != nil { + return errors.Wrap(err, "NewTask") + } + if _, err := db.Update(b, func() error { + b.TaskId = task.GetId() + b.State = api.LLMBenchmarkStatePending + return nil + }); err != nil { + return err + } + return task.ScheduleRun(nil) +} + +func (b *SLLMBenchmark) UpdateMetrics(ctx context.Context, userCred mcclient.TokenCredential, metrics *bench.LLMBenchmarkMetrics) error { + _, err := db.Update(b, func() error { + b.RequestsPerSecondMean = floatValue(metrics.RequestsPerSecondMean) + b.RequestLatencyMeanSec = floatValue(metrics.RequestLatencyMeanSec) + b.RequestTotal = metrics.RequestTotal + b.RequestSuccessful = metrics.RequestSuccessful + b.RequestErrored = metrics.RequestErrored + b.ErrorRate = floatValue(metrics.ErrorRate) + return nil + }) + return err +} + +func (b *SLLMBenchmark) UpdateDatasetPreflight( + ctx context.Context, + userCred mcclient.TokenCredential, + summary api.LLMBenchmarkDatasetPreflight, + resultPath, logPath string, +) error { + _, err := db.Update(b, func() error { + b.DatasetPreflight = &summary + b.PreflightResultJson = resultPath + b.PreflightLogPath = logPath + return nil + }) + return err +} + +func (b *SLLMBenchmark) UpdateDatasetEvaluation( + ctx context.Context, + userCred mcclient.TokenCredential, + summary api.LLMBenchmarkDatasetEvaluation, + resultJSON, resultCSV, logPath string, +) error { + _, err := db.Update(b, func() error { + b.DatasetEvaluation = &summary + b.EvaluationResultJson = resultJSON + b.EvaluationResultCsv = resultCSV + b.EvaluationLogPath = logPath + return nil + }) + return err +} + +func floatValue(v *float64) float64 { + if v == nil { + return 0 + } + return *v +} + +func (b *SLLMBenchmark) ArtifactPath(kind string) (string, error) { + switch kind { + case "preflight": + return b.PreflightResultJson, nil + case "preflight-log": + return b.PreflightLogPath, nil + case "log": + return b.LogPath, nil + case "json": + return b.ResultJson, nil + case "csv": + return b.ResultCsv, nil + case "evaluation": + return b.EvaluationResultJson, nil + case "evaluation-csv": + return b.EvaluationResultCsv, nil + case "evaluation-log": + return b.EvaluationLogPath, nil + default: + return "", httperrors.NewInputParameterError("unknown artifact type %s", kind) + } +} + +func (b *SLLMBenchmark) ArtifactLocations() map[string]string { + return map[string]string{ + "preflight": b.PreflightResultJson, + "preflight-log": b.PreflightLogPath, + "log": b.LogPath, + "json": b.ResultJson, + "csv": b.ResultCsv, + "evaluation": b.EvaluationResultJson, + "evaluation-csv": b.EvaluationResultCsv, + "evaluation-log": b.EvaluationLogPath, + } +} + +func applyLLMBenchmarkArtifactLocations(b *SLLMBenchmark, locations map[string]string) { + b.PreflightResultJson = locations["preflight"] + b.PreflightLogPath = locations["preflight-log"] + b.LogPath = locations["log"] + b.ResultJson = locations["json"] + b.ResultCsv = locations["csv"] + b.EvaluationResultJson = locations["evaluation"] + b.EvaluationResultCsv = locations["evaluation-csv"] + b.EvaluationLogPath = locations["evaluation-log"] +} + +func (b *SLLMBenchmark) UpdateArtifactLocations( + ctx context.Context, + userCred mcclient.TokenCredential, + locations map[string]string, + storage, message string, +) error { + _, err := db.Update(b, func() error { + applyLLMBenchmarkArtifactLocations(b, locations) + b.ArtifactStorage = storage + b.ArtifactStorageMessage = message + return nil + }) + return err +} + +func (b *SLLMBenchmark) CleanupArtifacts(ctx context.Context) error { + var firstErr error + if err := bench.DefaultArtifactStore().DeleteBenchmark(ctx, b.ProjectId, b.Id); err != nil { + firstErr = err + } + if b.WorkDir == "" { + return firstErr + } + if !benchmarkWorkDirIsSafe(benchmarkWorkDirRoot(), b.WorkDir) { + if firstErr != nil { + return firstErr + } + return errors.Errorf("unsafe benchmark workdir %s", b.WorkDir) + } + if err := os.RemoveAll(b.WorkDir); err != nil && firstErr == nil { + firstErr = err + } + return firstErr +} + +func (b *SLLMBenchmark) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error { + if utils.IsInStringArray(b.State, []string{api.LLMBenchmarkStatePending, api.LLMBenchmarkStateQueued, api.LLMBenchmarkStateValidating, api.LLMBenchmarkStateRunning}) { + return httperrors.NewInvalidStatusError("benchmark is %s, stop it first", b.State) + } + return nil +} + +func (b *SLLMBenchmark) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error { + return b.StartDeleteTask(ctx, userCred, "") +} + +func (b *SLLMBenchmark) StartDeleteTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error { + task, err := taskman.TaskManager.NewTask(ctx, "LLMBenchmarkDeleteTask", b, userCred, nil, parentTaskId, "", nil) + if err != nil { + return errors.Wrap(err, "NewTask LLMBenchmarkDeleteTask") + } + return task.ScheduleRun(nil) +} + +func (b *SLLMBenchmark) Delete(ctx context.Context, userCred mcclient.TokenCredential) error { + return nil +} + +func (b *SLLMBenchmark) RealDelete(ctx context.Context, userCred mcclient.TokenCredential) error { + return b.SVirtualResourceBase.Delete(ctx, userCred) +} + +func (b *SLLMBenchmark) PerformRetest(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.LLMBenchmarkRetestInput) (jsonutils.JSONObject, error) { + if err := validateBenchmarkMutableState(b.State); err != nil { + return nil, err + } + + createInput := benchmarkCreateInputFromModel(b) + createInput.BenchmarkImage = b.LLMImageId + createInput.BenchmarkPackage = b.BenchmarkPackageId + if b.LLMDeploymentId != "" { + createInput.LLMId = "" + } + prepared, err := prepareLLMBenchmarkCreateInput(ctx, userCred, b.GetOwnerId(), createInput) + if err != nil { + return nil, err + } + + if b.RunnerServerId != "" { + if err := b.DeleteRunnerServer(ctx, userCred); err != nil { + return nil, errors.Wrap(err, "delete old benchmark runner") + } + } + if err := b.CleanupArtifacts(ctx); err != nil { + log.Warningf("cleanup benchmark %s artifacts before retest: %s", b.Id, err) + } + + _, err = db.Update(b, func() error { + b.LLMId = prepared.LLMId + b.LLMDeploymentId = prepared.LLMDeploymentId + b.LLMSkuId = prepared.LLMSkuId + b.LLMImageId = prepared.LLMImageId + b.BenchmarkPackageId = prepared.BenchmarkPackageId + b.Backend = prepared.Backend + b.TargetUrl = prepared.TargetUrl + b.Model = prepared.Model + b.WorkDir = prepared.WorkDir + b.TargetSnapshot = prepared.TargetSnapshot + b.GuideLLMSpec = prepared.GuideLLMSpec + resetLLMBenchmarkResultFields(b) + return nil + }) + if err != nil { + return nil, errors.Wrap(err, "reset benchmark") + } + if err := b.StartRunTask(ctx, userCred, ""); err != nil { + _ = b.SetState(ctx, userCred, api.LLMBenchmarkStateError, err.Error()) + return nil, err + } + return nil, nil +} + +func requestLLMBenchmarkStop(b *SLLMBenchmark) { + b.StopRequested = true +} + +func (b *SLLMBenchmark) PerformStop(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + obj, err := GetLLMBenchmarkManager().FetchById(b.Id) + if err != nil { + return nil, err + } + current := obj.(*SLLMBenchmark) + if !utils.IsInStringArray(current.State, []string{api.LLMBenchmarkStatePending, api.LLMBenchmarkStateQueued, api.LLMBenchmarkStateValidating, api.LLMBenchmarkStateRunning}) { + return nil, httperrors.NewInvalidStatusError("benchmark is %s", current.State) + } + _, err = db.Update(current, func() error { + requestLLMBenchmarkStop(current) + return nil + }) + if err != nil { + return nil, err + } + stopLLMBenchmarkAsync(current.Id, userCred) + return nil, nil +} + +func stopLLMBenchmarkAsync(benchmarkId string, userCred mcclient.TokenCredential) { + go func() { + ctx := context.Background() + obj, err := GetLLMBenchmarkManager().FetchById(benchmarkId) + if err != nil { + log.Warningf("fetch benchmark %s for stop: %s", benchmarkId, err) + return + } + benchmark := obj.(*SLLMBenchmark) + if err := benchmark.DeleteRunnerServer(ctx, userCred); err != nil { + log.Warningf("delete benchmark %s runner: %s", benchmarkId, err) + } + }() +} + +func (b *SLLMBenchmark) DeleteRunnerServer(ctx context.Context, userCred mcclient.TokenCredential) error { + if b.RunnerServerId == "" { + return nil + } + s := auth.GetSession(ctx, userCred, options.Options.Region) + _, _ = compute.Servers.Update(s, b.RunnerServerId, jsonutils.Marshal(map[string]interface{}{ + "disable_delete": false, + })) + _, err := compute.Servers.DeleteWithParam(s, b.RunnerServerId, jsonutils.Marshal(map[string]interface{}{ + "override_pending_delete": true, + }), nil) + if err != nil && !isBenchmarkRunnerNotFound(err) { + return err + } + _, err = db.Update(b, func() error { + b.RunnerServerId = "" + b.RunnerContainerId = "" + return nil + }) + return err +} + +func isBenchmarkRunnerNotFound(err error) bool { + var clientErr *httputils.JSONClientError + return stderrors.As(err, &clientErr) && clientErr.Code == 404 +} + +func (b *SLLMBenchmark) ResolveTokenizerMount() (*LLMBenchmarkTokenizerMount, error) { + obj, err := GetLLMManager().FetchById(b.LLMId) + if err != nil { + return nil, errors.Wrap(err, "fetch benchmark LLM") + } + return resolveBenchmarkTokenizerMount(obj.(*SLLM), b.Model) +} + +func (b *SLLMBenchmark) RunnerPodInput(image string, server *computeapi.ServerDetails, tokenizer *LLMBenchmarkTokenizerMount) (*computeapi.ServerCreateInput, error) { + if len(server.Nics) == 0 || server.Nics[0].NetworkId == "" { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "target server network is empty") + } + cpu := options.Options.LLMBenchmarkRunnerCPU + if cpu <= 0 { + cpu = 1 + } + mem := options.Options.LLMBenchmarkRunnerMemoryMB + if mem <= 0 { + mem = 2048 + } + input := bench.RunnerPodInput{ + Name: "llm-bench-" + b.Id, + Image: image, + NetworkId: server.Nics[0].NetworkId, + HostId: server.HostId, + CPU: cpu, + MemoryMB: mem, + } + if b.BenchmarkPackageId != "" { + obj, err := GetLLMBenchmarkPackageManager().FetchById(b.BenchmarkPackageId) + if err != nil { + return nil, errors.Wrap(err, "fetch benchmark package") + } + pkg := obj.(*SLLMBenchmarkPackage) + input.PackageImageId = pkg.ImageId + input.PackageMountBase = api.LLMBenchmarkPackageMountBase + input.PackageSizeMB = int(pkg.ActualSizeMb) + 512 + } + if tokenizer != nil { + input.ModelImageId = tokenizer.ImageId + input.ModelSizeMB = tokenizer.SizeMB + input.ModelMountBase = tokenizer.MountBase + input.ModelMountSubdirectory = tokenizer.MountSubdirectory + } + return bench.BuildRunnerPodInput(input), nil +} diff --git a/pkg/llm/models/llm_benchmark_package.go b/pkg/llm/models/llm_benchmark_package.go new file mode 100644 index 0000000000..3999ce5586 --- /dev/null +++ b/pkg/llm/models/llm_benchmark_package.go @@ -0,0 +1,520 @@ +package models + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/httputils" + "yunion.io/x/sqlchemy" + + commonapis "yunion.io/x/onecloud/pkg/apis" + imageapi "yunion.io/x/onecloud/pkg/apis/image" + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/llm/options" + "yunion.io/x/onecloud/pkg/mcclient" + imagemodules "yunion.io/x/onecloud/pkg/mcclient/modules/image" +) + +func init() { + GetLLMBenchmarkPackageManager() +} + +var llmBenchmarkPackageManager *SLLMBenchmarkPackageManager + +func GetLLMBenchmarkPackageManager() *SLLMBenchmarkPackageManager { + if llmBenchmarkPackageManager != nil { + return llmBenchmarkPackageManager + } + llmBenchmarkPackageManager = &SLLMBenchmarkPackageManager{ + SSharableVirtualResourceBaseManager: db.NewSharableVirtualResourceBaseManager( + SLLMBenchmarkPackage{}, + "llm_benchmark_packages_tbl", + "llm_benchmark_package", + "llm_benchmark_packages", + ), + } + llmBenchmarkPackageManager.SetVirtualObject(llmBenchmarkPackageManager) + return llmBenchmarkPackageManager +} + +type SLLMBenchmarkPackageManager struct { + db.SSharableVirtualResourceBaseManager +} + +type SLLMBenchmarkPackage struct { + db.SSharableVirtualResourceBase + + Source string `width:"64" charset:"ascii" nullable:"false" default:"huggingface" list:"user" create:"optional"` + RepoId string `width:"256" charset:"utf8" nullable:"true" list:"user" create:"optional"` + Revision string `width:"128" charset:"utf8" nullable:"true" list:"user" create:"optional"` + FilePath string `width:"512" charset:"utf8" nullable:"true" list:"user" create:"optional"` + Format string `width:"64" charset:"ascii" nullable:"false" default:"guidellm_jsonl" list:"user" create:"optional"` + AnswerColumn string `width:"128" charset:"utf8" nullable:"true" list:"user" create:"optional"` + + ImageId string `width:"128" charset:"ascii" nullable:"true" list:"user" create:"optional"` + Size int64 `nullable:"true" list:"user" create:"optional"` + ActualSizeMb int32 `nullable:"true" list:"user" create:"optional"` + + MountPath string `charset:"utf8" length:"medium" nullable:"true" list:"user" create:"optional"` + DatasetPath string `charset:"utf8" length:"medium" nullable:"true" list:"user" create:"optional"` + Manifest string `charset:"utf8" length:"long" nullable:"true" list:"user" create:"optional"` +} + +func (man *SLLMBenchmarkPackageManager) ValidateCreateData( + ctx context.Context, + userCred mcclient.TokenCredential, + ownerId mcclient.IIdentityProvider, + query jsonutils.JSONObject, + input api.LLMBenchmarkPackageCreateInput, +) (api.LLMBenchmarkPackageCreateInput, error) { + var err error + input.SharableVirtualResourceCreateInput, err = man.SSharableVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.SharableVirtualResourceCreateInput) + if err != nil { + return input, errors.Wrap(err, "validate SharableVirtualResourceCreateInput") + } + defaultLLMBenchmarkPackageInput(&input) + if input.ImageId != "" { + img, err := fetchImage(ctx, userCred, input.ImageId) + if err != nil { + return input, errors.Wrap(err, "fetch image") + } + if img.DiskFormat != imageapi.IMAGE_DISK_FORMAT_TGZ { + return input, errors.Wrapf(httperrors.ErrInvalidFormat, "cannot use image of format %s", img.DiskFormat) + } + input.ImageId = img.Id + input.Size = img.Size + input.ActualSizeMb = img.MinDiskMB + input.Status = img.Status + return input, nil + } + if input.Source != api.LLMBenchmarkPackageSourceHuggingFace { + return input, errors.Wrap(httperrors.ErrInputParameter, "source only supports huggingface") + } + if input.Format != api.LLMBenchmarkPackageFormatGuideLLMJSONL { + return input, errors.Wrap(httperrors.ErrInputParameter, "format only supports guidellm_jsonl") + } + if strings.TrimSpace(input.RepoId) == "" { + return input, httperrors.NewMissingParameterError("repo_id") + } + if strings.TrimSpace(input.FilePath) == "" { + return input, httperrors.NewMissingParameterError("file_path") + } + input.Status = imageapi.IMAGE_STATUS_QUEUED + return input, nil +} + +func defaultLLMBenchmarkPackageInput(input *api.LLMBenchmarkPackageCreateInput) { + input.AnswerColumn = strings.TrimSpace(input.AnswerColumn) + if input.Source == "" { + input.Source = api.LLMBenchmarkPackageSourceHuggingFace + } + if input.Revision == "" { + input.Revision = "main" + } + if input.Format == "" { + input.Format = api.LLMBenchmarkPackageFormatGuideLLMJSONL + } + if input.MountPath == "" { + input.MountPath = path.Join(api.LLMBenchmarkPackageMountBase, benchmarkPackagePathName(input.Name)) + } + if input.DatasetPath == "" { + input.DatasetPath = path.Join(input.MountPath, "data.jsonl") + } +} + +func benchmarkPackagePathName(name string) string { + name = strings.TrimSpace(name) + var b strings.Builder + lastDash := false + for _, r := range name { + keep := (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' + if keep { + b.WriteRune(r) + lastDash = false + continue + } + if !lastDash { + b.WriteByte('-') + lastDash = true + } + } + out := strings.Trim(b.String(), "-") + if out == "" { + return "package" + } + return out +} + +func (man *SLLMBenchmarkPackageManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, input api.LLMBenchmarkPackageListInput) (*sqlchemy.SQuery, error) { + q, err := man.SSharableVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, input.SharableVirtualResourceListInput) + if err != nil { + return nil, errors.Wrap(err, "ListItemFilter") + } + if input.Source != "" { + q = q.Equals("source", input.Source) + } + if input.RepoId != "" { + q = q.Equals("repo_id", input.RepoId) + } + if input.Format != "" { + q = q.Equals("format", input.Format) + } + return q, nil +} + +func (man *SLLMBenchmarkPackageManager) PerformImport( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + input api.LLMBenchmarkPackageImportInput, +) (*SLLMBenchmarkPackage, error) { + input.ImageId = "" + data := jsonutils.Marshal(input) + obj, err := db.DoCreate(man, ctx, userCred, nil, data, userCred) + if err != nil { + return nil, errors.Wrap(err, "DoCreate") + } + pkg := obj.(*SLLMBenchmarkPackage) + if err := data.Unmarshal(&input); err != nil { + _ = pkg.SetStatus(ctx, userCred, imageapi.IMAGE_STATUS_KILLED, err.Error()) + return nil, errors.Wrap(err, "unmarshal validated import input") + } + if err := pkg.startImportTask(ctx, userCred, input, ""); err != nil { + _ = pkg.SetStatus(ctx, userCred, imageapi.IMAGE_STATUS_KILLED, err.Error()) + return nil, err + } + return pkg, nil +} + +func (pkg *SLLMBenchmarkPackage) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) { + pkg.SSharableVirtualResourceBase.PostCreate(ctx, userCred, ownerId, query, data) + if pkg.ImageId != "" { + return + } + input := api.LLMBenchmarkPackageImportInput{} + if err := data.Unmarshal(&input); err != nil { + _ = pkg.SetStatus(ctx, userCred, imageapi.IMAGE_STATUS_KILLED, err.Error()) + return + } + if err := pkg.startImportTask(ctx, userCred, input, ""); err != nil { + _ = pkg.SetStatus(ctx, userCred, imageapi.IMAGE_STATUS_KILLED, err.Error()) + } +} + +func (pkg *SLLMBenchmarkPackage) startImportTask(ctx context.Context, userCred mcclient.TokenCredential, input api.LLMBenchmarkPackageImportInput, parentTaskId string) error { + params := jsonutils.NewDict() + params.Add(jsonutils.Marshal(input), "import_input") + task, err := taskman.TaskManager.NewTask(ctx, "LLMBenchmarkPackageImportTask", pkg, userCred, params, parentTaskId, "") + if err != nil { + return errors.Wrap(err, "NewTask") + } + return task.ScheduleRun(nil) +} + +func (pkg *SLLMBenchmarkPackage) updateImportStatus(ctx context.Context, userCred mcclient.TokenCredential, status, reason string) error { + if pkg.Status == status { + return nil + } + _, err := db.Update(pkg, func() error { + pkg.Status = status + return nil + }) + return errors.Wrap(err, reason) +} + +func (pkg *SLLMBenchmarkPackage) DoImport(ctx context.Context, userCred mcclient.TokenCredential, s *mcclient.ClientSession, input api.LLMBenchmarkPackageCreateInput) (string, error) { + _ = pkg.SetProgress(0) + tmpDir, err := getLLMBenchmarkPackageImportWorkDir(options.Options.LLMWorkingDirectory, input) + if err != nil { + return "", err + } + if err := os.MkdirAll(tmpDir, 0755); err != nil { + return "", errors.Wrap(err, "mkdir import work dir") + } + rootDir := filepath.Join(tmpDir, "package") + dataDir := filepath.Join(rootDir, "data") + if err := os.MkdirAll(dataDir, 0755); err != nil { + return tmpDir, errors.Wrap(err, "mkdir data dir") + } + if err := downloadHuggingFaceBenchmarkPackageFile(ctx, filepath.Join(dataDir, "data.jsonl"), input); err != nil { + return tmpDir, errors.Wrap(err, "download benchmark package") + } + _ = pkg.SetProgress(90) + + manifest := map[string]string{ + "source": input.Source, + "repo_id": input.RepoId, + "revision": input.Revision, + "file_path": input.FilePath, + "format": input.Format, + "dataset_path": pkg.DatasetPath, + "answer_column": input.AnswerColumn, + } + manifestStr := jsonutils.Marshal(manifest).String() + if err := os.WriteFile(filepath.Join(rootDir, "manifest.json"), []byte(manifestStr), 0644); err != nil { + return tmpDir, errors.Wrap(err, "write manifest") + } + archivePath := filepath.Join(tmpDir, "package.tgz") + if err := createTarGz(rootDir, archivePath); err != nil { + return tmpDir, errors.Wrap(err, "createTarGz") + } + _ = pkg.SetProgress(95) + + if err := pkg.updateImportStatus(ctx, userCred, imageapi.IMAGE_STATUS_SAVING, "saving"); err != nil { + return tmpDir, err + } + imageId, imgSize, err := pkg.uploadImage(ctx, userCred, s, input, archivePath, manifestStr) + if err != nil { + return tmpDir, errors.Wrap(err, "upload image") + } + _ = pkg.SetProgress(98) + + _, err = db.Update(pkg, func() error { + pkg.ImageId = imageId + pkg.Size = imgSize + pkg.Manifest = manifestStr + return nil + }) + if err != nil { + return tmpDir, errors.Wrap(err, "update benchmark package") + } + if _, err := pkg.WaitImageStatus(ctx, userCred, []string{imageapi.IMAGE_STATUS_ACTIVE}, 1800); err != nil { + return tmpDir, errors.Wrap(err, "wait image") + } + if err := pkg.syncImageStatus(ctx, userCred); err != nil { + return tmpDir, errors.Wrap(err, "sync image") + } + _ = pkg.SetProgress(100) + return tmpDir, nil +} + +func getLLMBenchmarkPackageImportWorkDir(root string, input api.LLMBenchmarkPackageCreateInput) (string, error) { + root = strings.TrimSpace(root) + if root == "" { + return "", errors.Error("LLMWorkingDirectory is empty") + } + key := strings.Join([]string{input.Source, input.RepoId, input.Revision, input.FilePath, input.Format, input.AnswerColumn}, "\x00") + sum := sha256.Sum256([]byte(key)) + display := benchmarkPackagePathName(input.Name) + return filepath.Join(root, "benchmark-package-import-cache", fmt.Sprintf("%s-%s", display, hex.EncodeToString(sum[:])[:16])), nil +} + +func (pkg *SLLMBenchmarkPackage) uploadImage(ctx context.Context, userCred mcclient.TokenCredential, s *mcclient.ClientSession, input api.LLMBenchmarkPackageCreateInput, archivePath string, manifest string) (string, int64, error) { + f, err := os.Open(archivePath) + if err != nil { + return "", 0, errors.Wrap(err, "open archive") + } + defer f.Close() + stat, err := f.Stat() + if err != nil { + return "", 0, errors.Wrap(err, "stat archive") + } + size := stat.Size() + protected := false + imgParams := imageapi.ImageCreateInput{} + imgParams.GenerateName = pkg.Name + imgParams.DiskFormat = imageapi.IMAGE_DISK_FORMAT_TGZ + imgParams.Size = &size + imgParams.Protected = &protected + imgParams.Properties = map[string]string{ + "llm_benchmark_package": "true", + "source": input.Source, + "source_repo_id": input.RepoId, + "source_requested_revision": input.Revision, + "source_file_path": input.FilePath, + "format": input.Format, + "answer_column": input.AnswerColumn, + "manifest": manifest, + imageapi.IMAGE_INTERNAL_PATH_MAP: jsonutils.Marshal(map[string]string{"data": pkg.MountPath}).String(), + imageapi.IMAGE_USED_BY_POST_OVERLAY: "true", + } + imageObj, err := imagemodules.Images.Upload(s, jsonutils.Marshal(imgParams), f, size) + if err != nil { + return "", 0, errors.Wrap(err, "Upload") + } + imageId, err := imageObj.GetString("id") + if err != nil { + return "", 0, errors.Wrap(err, "GetString id") + } + return imageId, size, nil +} + +func (pkg *SLLMBenchmarkPackage) syncImageStatus(ctx context.Context, userCred mcclient.TokenCredential) error { + img, err := fetchImage(ctx, userCred, pkg.ImageId) + if err != nil { + return err + } + _, err = db.Update(pkg, func() error { + pkg.Status = img.Status + pkg.Size = img.Size + pkg.ActualSizeMb = img.MinDiskMB + return nil + }) + return err +} + +func (pkg *SLLMBenchmarkPackage) WaitImageStatus(ctx context.Context, userCred mcclient.TokenCredential, targetStatus []string, timeoutSecs int) (*imageapi.ImageDetails, error) { + expire := time.Now().Add(time.Second * time.Duration(timeoutSecs)) + for time.Now().Before(expire) { + img, err := fetchImage(ctx, userCred, pkg.ImageId) + if err != nil { + return nil, err + } + for _, status := range targetStatus { + if img.Status == status { + return img, nil + } + } + if img.Status == imageapi.IMAGE_STATUS_KILLED || img.Status == imageapi.IMAGE_STATUS_DEACTIVATED { + return nil, errors.Wrap(errors.ErrInvalidStatus, img.Status) + } + time.Sleep(2 * time.Second) + } + return nil, errors.Wrapf(httperrors.ErrTimeout, "wait image status %s timeout", targetStatus) +} + +func (pkg *SLLMBenchmarkPackage) CleanupImportTmpDir(ctx context.Context, userCred mcclient.TokenCredential, dir string) { + if dir == "" { + return + } + if err := os.RemoveAll(dir); err != nil { + log.Warningf("cleanup benchmark package import dir %s: %s", dir, err) + } +} + +func downloadHuggingFaceBenchmarkPackageFile(ctx context.Context, dst string, input api.LLMBenchmarkPackageCreateInput) error { + endpoint := strings.TrimRight(options.Options.HuggingFaceEndpoint, "/") + if endpoint == "" { + endpoint = huggingFaceMirrorEndpoint + } + fileURL := buildHuggingFaceDatasetFileURL(endpoint, input.RepoId, input.Revision, input.FilePath) + if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { + return err + } + tmp := dst + ".tmp" + f, err := os.Create(tmp) + if err != nil { + return errors.Wrap(err, "create tmp") + } + client := httputils.GetTimeoutClient(0) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, fileURL, nil) + if err != nil { + _ = f.Close() + return err + } + if options.Options.HuggingFaceToken != "" { + req.Header.Set("Authorization", "Bearer "+options.Options.HuggingFaceToken) + } + resp, err := client.Do(req) + if err != nil { + _ = f.Close() + return err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + _ = f.Close() + return errors.Errorf("download %s failed: %s", fileURL, resp.Status) + } + if _, err := io.Copy(f, resp.Body); err != nil { + _ = f.Close() + return err + } + if err := f.Close(); err != nil { + return err + } + return os.Rename(tmp, dst) +} + +func buildHuggingFaceDatasetFileURL(endpoint, repoID, revision, filePath string) string { + return fmt.Sprintf("%s/datasets/%s/resolve/%s/%s", + strings.TrimRight(endpoint, "/"), + escapeURLPathPreserveSlash(repoID), + url.PathEscape(revision), + escapeURLPathPreserveSlash(filePath), + ) +} + +func validateBenchmarkPackageDeleteStatus(status string) error { + if status == imageapi.IMAGE_STATUS_SAVING || status == imageapi.IMAGE_STATUS_QUEUED || status == commonapis.STATUS_DELETING { + return httperrors.NewInvalidStatusError("benchmark package is %s", status) + } + return nil +} + +func (pkg *SLLMBenchmarkPackage) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error { + if err := validateBenchmarkPackageDeleteStatus(pkg.Status); err != nil { + return err + } + if err := ValidateBenchmarkPackageUnused(pkg.Id, ""); err != nil { + return err + } + return pkg.SSharableVirtualResourceBase.ValidateDeleteCondition(ctx, info) +} + +func (pkg *SLLMBenchmarkPackage) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error { + manager := GetLLMBenchmarkManager() + lockKey := db.GetLockClassKey(manager, pkg.GetOwnerId()) + lockman.LockClass(ctx, manager, lockKey) + defer lockman.ReleaseClass(ctx, manager, lockKey) + + if err := ValidateBenchmarkPackageUnused(pkg.Id, ""); err != nil { + return err + } + return pkg.StartDeleteTask(ctx, userCred, query, "") +} + +func (pkg *SLLMBenchmarkPackage) StartDeleteTask(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, parentTaskId string) error { + if err := validateBenchmarkPackageDeleteStatus(pkg.Status); err != nil { + return err + } + previousStatus := pkg.Status + if err := pkg.SetStatus(ctx, userCred, commonapis.STATUS_DELETING, ""); err != nil { + return errors.Wrap(err, "set benchmark package deleting") + } + rollbackStatus := func(startErr error) error { + if err := pkg.SetStatus(ctx, userCred, previousStatus, startErr.Error()); err != nil { + return errors.Wrapf(startErr, "rollback benchmark package status: %s", err) + } + return startErr + } + params := jsonutils.NewDict() + if pkg.ImageId != "" { + params.Set("image_id", jsonutils.NewString(pkg.ImageId)) + } + if jsonutils.QueryBoolean(query, "purge", false) { + params.Set("purge", jsonutils.JSONTrue) + } + task, err := taskman.TaskManager.NewTask(ctx, "LLMBenchmarkPackageDeleteTask", pkg, userCred, params, parentTaskId, "") + if err != nil { + return rollbackStatus(errors.Wrap(err, "NewTask LLMBenchmarkPackageDeleteTask")) + } + if err := task.ScheduleRun(nil); err != nil { + return rollbackStatus(errors.Wrap(err, "ScheduleRun LLMBenchmarkPackageDeleteTask")) + } + return nil +} + +func (pkg *SLLMBenchmarkPackage) Delete(ctx context.Context, userCred mcclient.TokenCredential) error { + return nil +} + +func (pkg *SLLMBenchmarkPackage) RealDelete(ctx context.Context, userCred mcclient.TokenCredential) error { + return pkg.SSharableVirtualResourceBase.Delete(ctx, userCred) +} diff --git a/pkg/llm/models/llm_benchmark_stop_state.go b/pkg/llm/models/llm_benchmark_stop_state.go new file mode 100644 index 0000000000..0440be8747 --- /dev/null +++ b/pkg/llm/models/llm_benchmark_stop_state.go @@ -0,0 +1,13 @@ +package models + +import api "yunion.io/x/onecloud/pkg/apis/llm" + +func benchmarkRunFinalState(state string, stopRequested bool, runErr error) (string, string) { + if stopRequested || state == api.LLMBenchmarkStateStopped { + return api.LLMBenchmarkStateStopped, "" + } + if runErr != nil { + return api.LLMBenchmarkStateError, runErr.Error() + } + return api.LLMBenchmarkStateCompleted, "" +} diff --git a/pkg/llm/options/option.go b/pkg/llm/options/option.go index b87d9c2560..6647591ffb 100644 --- a/pkg/llm/options/option.go +++ b/pkg/llm/options/option.go @@ -29,6 +29,24 @@ type LLMOptions struct { ImportTaskWorkerCount int `help:"import task worker count" default:"8"` StartTaskWorkerCount int `help:"start task worker count" default:"128"` + LLMBenchmarkWorkDir string `help:"llm benchmark working directory" default:"/opt/cloud/workspace/llm/benchmarks"` + LLMBenchmarkDefaultImage string `help:"default GuideLLM benchmark image" default:"registry.cn-beijing.aliyuncs.com/cloudpods/guidellm:v0.7.0-amd64"` + LLMBenchmarkRunnerCPU int `help:"llm benchmark runner cpu" default:"1"` + LLMBenchmarkRunnerMemoryMB int `help:"llm benchmark runner memory MB" default:"2048"` + LLMBenchmarkDefaultRequestRate int `help:"default benchmark request rate" default:"1"` + LLMBenchmarkDefaultTotalRequests int `help:"default benchmark total requests" default:"100"` + LLMBenchmarkDefaultInputTokens int `help:"default synthetic prompt tokens" default:"1024"` + LLMBenchmarkDefaultOutputTokens int `help:"default synthetic output tokens" default:"128"` + LLMBenchmarkMaxDurationSeconds int `help:"max benchmark duration seconds" default:"3600"` + LLMBenchmarkMaxRequestRate int `help:"max benchmark request rate" default:"100"` + LLMBenchmarkMaxTotalRequests int `help:"max benchmark total requests" default:"100000"` + ArtifactS3Endpoint string `help:"MinIO/S3 endpoint for benchmark artifacts; empty disables upload" default:"http://monitor-minio.onecloud-monitoring.svc:9000"` + ArtifactS3AccessKey string `help:"MinIO/S3 access key for benchmark artifacts" default:"monitor-admin"` + ArtifactS3SecretKey string `help:"MinIO/S3 secret key for benchmark artifacts" default:"monitor-admin"` + ArtifactS3Bucket string `help:"MinIO/S3 bucket for benchmark artifacts" default:"llm-benchmark"` + ArtifactS3Secure bool `help:"Use HTTPS for benchmark artifact endpoint without scheme" default:"false"` + ArtifactS3Prefix string `help:"MinIO/S3 object key prefix for benchmark artifacts" default:"llm-benchmarks"` + // MCP Agent 配置 MCPServerURL string `help:"MCP Server URL" default:"http://default-mcp-server:30876"` MCPAgentTimeout int `help:"MCP Agent request timeout in seconds" default:"120"` diff --git a/pkg/llm/service/benchmark_handler.go b/pkg/llm/service/benchmark_handler.go new file mode 100644 index 0000000000..74d4f988a3 --- /dev/null +++ b/pkg/llm/service/benchmark_handler.go @@ -0,0 +1,214 @@ +package service + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + + "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/appsrv" + "yunion.io/x/onecloud/pkg/cloudcommon/policy" + "yunion.io/x/onecloud/pkg/httperrors" + bench "yunion.io/x/onecloud/pkg/llm/benchmark" + "yunion.io/x/onecloud/pkg/llm/models" + "yunion.io/x/onecloud/pkg/mcclient/auth" +) + +func AddBenchmarkArtifactHandlers(app *appsrv.Application) { + app.AddHandler2("GET", "/llm_benchmarks//artifacts", auth.Authenticate(handleLLMBenchmarkArtifacts), nil, "llm_benchmark_artifacts", nil) + app.AddHandler2("GET", "/llm_benchmarks//artifacts/", auth.Authenticate(handleLLMBenchmarkArtifact), nil, "llm_benchmark_artifact", nil) +} + +type benchmarkArtifactItem struct { + Type string `json:"type"` + Filename string `json:"filename"` + Ready bool `json:"ready"` +} + +func handleLLMBenchmarkArtifacts(ctx context.Context, w http.ResponseWriter, r *http.Request) { + b, ok := fetchLLMBenchmarkForArtifact(ctx, w, benchmarkArtifactParam(ctx, "")) + if !ok { + return + } + items, err := buildBenchmarkArtifactList(ctx, b, bench.DefaultArtifactStore()) + if err != nil { + httperrors.GeneralServerError(ctx, w, err) + return + } + appsrv.SendJSON(w, jsonutils.Marshal(map[string]interface{}{ + "artifacts": items, + })) +} + +func handleLLMBenchmarkArtifact(ctx context.Context, w http.ResponseWriter, r *http.Request) { + id := benchmarkArtifactParam(ctx, "") + kind := benchmarkArtifactParam(ctx, "") + if kind == "" { + httperrors.InvalidInputError(ctx, w, "missing artifact type") + return + } + sendBenchmarkArtifact(ctx, w, r, id, kind) +} + +func sendBenchmarkArtifact(ctx context.Context, w http.ResponseWriter, r *http.Request, id, kind string) { + b, ok := fetchLLMBenchmarkForArtifact(ctx, w, id) + if !ok { + return + } + found, err := writeBenchmarkArtifact(ctx, w, r, b, kind, bench.DefaultArtifactStore()) + if err != nil { + httperrors.GeneralServerError(ctx, w, err) + return + } + if !found { + httperrors.NotFoundError(ctx, w, "artifact %s not found", kind) + } +} + +func setBenchmarkArtifactHeaders(w http.ResponseWriter, kind string) { + w.Header().Set("Content-Type", artifactContentType(kind)) + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", artifactFilename(kind))) +} + +func writeBenchmarkArtifact( + ctx context.Context, + w http.ResponseWriter, + r *http.Request, + b *models.SLLMBenchmark, + kind string, + store *bench.ArtifactStore, +) (bool, error) { + location, err := b.ArtifactPath(kind) + if err != nil { + return false, err + } + if location != "" && !strings.HasPrefix(location, "s3://") { + if ready, err := store.Exists(ctx, location); err != nil { + return false, err + } else if ready { + setBenchmarkArtifactHeaders(w, kind) + http.ServeFile(w, r, location) + return true, nil + } + } + if strings.HasPrefix(location, "s3://") { + body, err := store.Open(ctx, location) + if err != nil { + return false, err + } + defer body.Close() + setBenchmarkArtifactHeaders(w, kind) + _, err = io.Copy(w, body) + return true, err + } + raw := artifactRaw(b, kind) + if raw == "" { + return false, nil + } + setBenchmarkArtifactHeaders(w, kind) + _, err = io.WriteString(w, raw) + return true, err +} + +func benchmarkArtifactParam(ctx context.Context, key string) string { + params := appsrv.AppContextGetParams(ctx) + return params.Params[key] +} + +func fetchLLMBenchmarkForArtifact(ctx context.Context, w http.ResponseWriter, id string) (*models.SLLMBenchmark, bool) { + userCred := auth.FetchUserCredential(ctx, policy.FilterPolicyCredential) + if userCred == nil { + httperrors.UnauthorizedError(ctx, w, "Unauthorized") + return nil, false + } + obj, err := models.GetLLMBenchmarkManager().FetchByIdOrName(ctx, userCred, id) + if err != nil { + httperrors.GeneralServerError(ctx, w, err) + return nil, false + } + return obj.(*models.SLLMBenchmark), true +} + +func buildBenchmarkArtifactList( + ctx context.Context, + b *models.SLLMBenchmark, + store *bench.ArtifactStore, +) ([]benchmarkArtifactItem, error) { + kinds := []string{ + "preflight", "preflight-log", "log", "json", "csv", + "evaluation", "evaluation-csv", "evaluation-log", + } + ret := make([]benchmarkArtifactItem, 0, len(kinds)) + for _, kind := range kinds { + location, err := b.ArtifactPath(kind) + if err != nil { + return nil, err + } + ready, err := store.Exists(ctx, location) + if err != nil { + return nil, err + } + if !ready { + ready = artifactRaw(b, kind) != "" + } + ret = append(ret, benchmarkArtifactItem{ + Type: kind, + Filename: artifactFilename(kind), + Ready: ready, + }) + } + return ret, nil +} + +func artifactRaw(b *models.SLLMBenchmark, kind string) string { + switch kind { + case "preflight": + return b.RawPreflightResult + case "preflight-log": + return b.RawPreflightLog + case "log": + return b.RawLog + case "json": + return b.RawMetrics + case "csv": + return b.RawCsv + default: + return "" + } +} + +func artifactFilename(kind string) string { + switch kind { + case "preflight": + return "dataset-preflight.json" + case "preflight-log": + return "dataset-preflight.log" + case "log": + return "guidellm.log" + case "json": + return "benchmarks.json" + case "csv": + return "benchmarks.csv" + case "evaluation": + return "evaluation.json" + case "evaluation-csv": + return "evaluation.csv" + case "evaluation-log": + return "evaluation.log" + default: + return kind + } +} + +func artifactContentType(kind string) string { + switch kind { + case "json", "preflight", "evaluation": + return "application/json" + case "csv", "evaluation-csv": + return "text/csv" + default: + return "text/plain; charset=utf-8" + } +} diff --git a/pkg/llm/service/handler.go b/pkg/llm/service/handler.go index 82ea0f71f8..419af94584 100644 --- a/pkg/llm/service/handler.go +++ b/pkg/llm/service/handler.go @@ -328,6 +328,7 @@ func InitHandlers(app *appsrv.Application, isSlave bool) { app.AddHandler2("GET", "/llm_images_catalogs/", auth.Authenticate(handleLLMImagesCatalogShow), nil, "llm_images_catalog_show", nil) AddAvailableNetworkHandler(models.GetLLMManager().KeywordPlural(), app) + AddBenchmarkArtifactHandlers(app) // 默认 Agent 聊天流：优先于 dispatcher 注册，避免被 performClassAction 的 sendJSON 覆盖。 // 注册两种路径：default-chat-stream（apigateway 转发用）与 default/chat-stream（climc 直连 region 时用，否则会被当作 resid=default 的 perform 导致 404） @@ -365,6 +366,8 @@ func InitHandlers(app *appsrv.Application, isSlave bool) { models.GetLLMContainerManager(), models.GetLLMDeploymentManager(), models.GetLLMManager(), + models.GetLLMBenchmarkManager(), + models.GetLLMBenchmarkPackageManager(), // models.GetDifyManager(), models.GetInstantModelManager(), models.GetLLMInstantModelManager(), diff --git a/pkg/llm/service/service.go b/pkg/llm/service/service.go index d786a207f5..cbfad6f885 100644 --- a/pkg/llm/service/service.go +++ b/pkg/llm/service/service.go @@ -13,6 +13,7 @@ import ( app_common "yunion.io/x/onecloud/pkg/cloudcommon/app" "yunion.io/x/onecloud/pkg/cloudcommon/db" common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" + bench "yunion.io/x/onecloud/pkg/llm/benchmark" _ "yunion.io/x/onecloud/pkg/llm/drivers/llm_client" _ "yunion.io/x/onecloud/pkg/llm/drivers/llm_container" "yunion.io/x/onecloud/pkg/llm/models" @@ -28,6 +29,14 @@ func StartService() { dbOpts := &options.Options.DBOptions baseOpts := &opts.BaseOptions common_options.ParseOptions(opts, os.Args, "llm.conf", api.SERVICE_TYPE) + bench.ConfigureArtifactStore(bench.ArtifactStoreOptions{ + Endpoint: opts.ArtifactS3Endpoint, + AccessKey: opts.ArtifactS3AccessKey, + SecretKey: opts.ArtifactS3SecretKey, + Bucket: opts.ArtifactS3Bucket, + Secure: opts.ArtifactS3Secure, + Prefix: opts.ArtifactS3Prefix, + }) llmTask.InitInstantModelSyncTaskManager() app_common.InitAuth(commonOpts, func() { diff --git a/pkg/llm/tasks/llm/llm_benchmark_delete_task.go b/pkg/llm/tasks/llm/llm_benchmark_delete_task.go new file mode 100644 index 0000000000..10c92d09d3 --- /dev/null +++ b/pkg/llm/tasks/llm/llm_benchmark_delete_task.go @@ -0,0 +1,96 @@ +package llm + +import ( + "context" + "database/sql" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/llm/models" + "yunion.io/x/onecloud/pkg/util/logclient" +) + +type LLMBenchmarkDeleteTask struct { + taskman.STask +} + +func init() { + taskman.RegisterTask(LLMBenchmarkDeleteTask{}) +} + +func (task *LLMBenchmarkDeleteTask) taskFailed(ctx context.Context, benchmark *models.SLLMBenchmark, err error) { + _ = benchmark.SetState(ctx, task.UserCred, benchmark.State, err.Error()) + db.OpsLog.LogEvent(benchmark, db.ACT_DELETE_FAIL, err, task.UserCred) + logclient.AddActionLogWithStartable(task, benchmark, logclient.ACT_DELETE, err, task.UserCred, false) + task.SetStageFailed(ctx, jsonutils.NewString(err.Error())) +} + +func (task *LLMBenchmarkDeleteTask) deleteBenchmark(ctx context.Context, benchmark *models.SLLMBenchmark) { + if err := benchmark.CleanupArtifacts(ctx); err != nil { + log.Warningf("cleanup benchmark %s artifacts during delete: %s", benchmark.Id, err) + } + if err := benchmark.RealDelete(ctx, task.UserCred); err != nil { + task.taskFailed(ctx, benchmark, err) + return + } + logclient.AddActionLogWithStartable(task, benchmark, logclient.ACT_DELETE, nil, task.UserCred, true) + task.SetStageComplete(ctx, nil) +} + +func benchmarkPackageDeleteRequired(otherReferences int) bool { + return otherReferences == 0 +} + +func (task *LLMBenchmarkDeleteTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + benchmark := obj.(*models.SLLMBenchmark) + packageID := benchmark.BenchmarkPackageId + if packageID == "" { + task.deleteBenchmark(ctx, benchmark) + return + } + + manager := models.GetLLMBenchmarkManager() + lockKey := db.GetLockClassKey(manager, benchmark.GetOwnerId()) + lockman.LockClass(ctx, manager, lockKey) + defer lockman.ReleaseClass(ctx, manager, lockKey) + + otherReferences, err := models.CountBenchmarkPackageReferences(packageID, benchmark.Id) + if err != nil { + task.taskFailed(ctx, benchmark, err) + return + } + if !benchmarkPackageDeleteRequired(otherReferences) { + task.deleteBenchmark(ctx, benchmark) + return + } + + pkgObj, err := models.GetLLMBenchmarkPackageManager().FetchById(packageID) + if err != nil { + cause := errors.Cause(err) + if cause == errors.ErrNotFound || cause == sql.ErrNoRows { + task.deleteBenchmark(ctx, benchmark) + return + } + task.taskFailed(ctx, benchmark, errors.Wrap(err, "fetch benchmark package")) + return + } + pkg := pkgObj.(*models.SLLMBenchmarkPackage) + + task.SetStage("OnPackageDeleted", nil) + if err := pkg.StartDeleteTask(ctx, task.UserCred, nil, task.GetTaskId()); err != nil { + task.taskFailed(ctx, benchmark, errors.Wrap(err, "start package delete task")) + } +} + +func (task *LLMBenchmarkDeleteTask) OnPackageDeleted(ctx context.Context, benchmark *models.SLLMBenchmark, body jsonutils.JSONObject) { + task.deleteBenchmark(ctx, benchmark) +} + +func (task *LLMBenchmarkDeleteTask) OnPackageDeletedFailed(ctx context.Context, benchmark *models.SLLMBenchmark, body jsonutils.JSONObject) { + task.taskFailed(ctx, benchmark, errors.Errorf("benchmark package delete failed: %s", body)) +} diff --git a/pkg/llm/tasks/llm/llm_benchmark_package_delete_task.go b/pkg/llm/tasks/llm/llm_benchmark_package_delete_task.go new file mode 100644 index 0000000000..f04766fb83 --- /dev/null +++ b/pkg/llm/tasks/llm/llm_benchmark_package_delete_task.go @@ -0,0 +1,139 @@ +package llm + +import ( + "context" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/httputils" + + commonapis "yunion.io/x/onecloud/pkg/apis" + imageapi "yunion.io/x/onecloud/pkg/apis/image" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/llm/models" + "yunion.io/x/onecloud/pkg/llm/options" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/auth" + imagemodules "yunion.io/x/onecloud/pkg/mcclient/modules/image" + "yunion.io/x/onecloud/pkg/util/logclient" +) + +const ( + benchmarkPackageDeletePollInterval = 30 * time.Second + benchmarkPackageDeletePollMaxAttempts = 60 +) + +type LLMBenchmarkPackageDeleteTask struct { + taskman.STask +} + +func init() { + taskman.RegisterTask(LLMBenchmarkPackageDeleteTask{}) +} + +func benchmarkPackageImageNotFound(err error) bool { + if err == nil { + return false + } + if httputils.ErrorCode(err) == 404 { + return true + } + return strings.Contains(err.Error(), "ResourceNotFoundError") +} + +func (task *LLMBenchmarkPackageDeleteTask) taskFailed(ctx context.Context, pkg *models.SLLMBenchmarkPackage, err error) { + _ = pkg.SetStatus(ctx, task.UserCred, commonapis.STATUS_DELETE_FAILED, err.Error()) + db.OpsLog.LogEvent(pkg, db.ACT_DELETE_FAIL, err, task.UserCred) + logclient.AddActionLogWithStartable(task, pkg, logclient.ACT_DELETE, err, task.UserCred, false) + task.SetStageFailed(ctx, jsonutils.NewString(err.Error())) +} + +func (task *LLMBenchmarkPackageDeleteTask) imageID(pkg *models.SLLMBenchmarkPackage) string { + if imageID, _ := task.Params.GetString("image_id"); imageID != "" { + return imageID + } + return pkg.ImageId +} + +func (task *LLMBenchmarkPackageDeleteTask) waitImageDeleted(ctx context.Context, imageID string) error { + session := auth.GetAdminSession(ctx, options.Options.Region) + var lastErr error + for i := 0; i < benchmarkPackageDeletePollMaxAttempts; i++ { + _, err := imagemodules.Images.Get(session, imageID, nil) + if benchmarkPackageImageNotFound(err) { + return nil + } + if err != nil { + lastErr = err + } + time.Sleep(benchmarkPackageDeletePollInterval) + } + if lastErr != nil { + return errors.Wrapf(lastErr, "wait glance image %s deleted", imageID) + } + return errors.Errorf("wait glance image %s deleted timeout", imageID) +} + +func (task *LLMBenchmarkPackageDeleteTask) deleteImage(ctx context.Context, session *mcclient.ClientSession, imageID string) error { + protected := false + _, err := imagemodules.Images.Update(session, imageID, jsonutils.Marshal(imageapi.ImageUpdateInput{ + Protected: &protected, + })) + if err != nil && !benchmarkPackageImageNotFound(err) { + return errors.Wrapf(err, "unprotect glance image %s", imageID) + } + + params := jsonutils.NewDict() + params.Set("override_pending_delete", jsonutils.JSONTrue) + if jsonutils.QueryBoolean(task.Params, "purge", false) { + params.Set("purge", jsonutils.JSONTrue) + } + _, err = imagemodules.Images.DeleteWithParam(session, imageID, params, nil) + if err != nil && !benchmarkPackageImageNotFound(err) { + return errors.Wrapf(err, "delete glance image %s", imageID) + } + return task.waitImageDeleted(ctx, imageID) +} + +func (task *LLMBenchmarkPackageDeleteTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + pkg := obj.(*models.SLLMBenchmarkPackage) + _ = pkg.SetStatus(ctx, task.UserCred, commonapis.STATUS_DELETING, "start delete") + + imageID := task.imageID(pkg) + if imageID == "" { + task.OnImageDeleted(ctx, pkg, nil) + return + } + + session := auth.GetAdminSession(ctx, options.Options.Region) + _, err := imagemodules.Images.Get(session, imageID, nil) + if benchmarkPackageImageNotFound(err) { + task.OnImageDeleted(ctx, pkg, nil) + return + } + if err != nil { + task.taskFailed(ctx, pkg, errors.Wrapf(err, "get glance image %s", imageID)) + return + } + + task.SetStage("OnImageDeleted", nil) + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + return nil, task.deleteImage(ctx, session, imageID) + }) +} + +func (task *LLMBenchmarkPackageDeleteTask) OnImageDeletedFailed(ctx context.Context, pkg *models.SLLMBenchmarkPackage, body jsonutils.JSONObject) { + task.taskFailed(ctx, pkg, errors.Error(body.String())) +} + +func (task *LLMBenchmarkPackageDeleteTask) OnImageDeleted(ctx context.Context, pkg *models.SLLMBenchmarkPackage, body jsonutils.JSONObject) { + if err := pkg.RealDelete(ctx, task.UserCred); err != nil { + task.taskFailed(ctx, pkg, err) + return + } + logclient.AddActionLogWithStartable(task, pkg, logclient.ACT_DELETE, nil, task.UserCred, true) + task.SetStageComplete(ctx, nil) +} diff --git a/pkg/llm/tasks/llm/llm_benchmark_package_import_task.go b/pkg/llm/tasks/llm/llm_benchmark_package_import_task.go new file mode 100644 index 0000000000..2378007cdf --- /dev/null +++ b/pkg/llm/tasks/llm/llm_benchmark_package_import_task.go @@ -0,0 +1,97 @@ +package llm + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + imageapi "yunion.io/x/onecloud/pkg/apis/image" + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/llm/models" + "yunion.io/x/onecloud/pkg/llm/options" + "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/util/logclient" +) + +type LLMBenchmarkPackageImportTask struct { + taskman.STask +} + +func init() { + taskman.RegisterTask(LLMBenchmarkPackageImportTask{}) +} + +func (task *LLMBenchmarkPackageImportTask) taskFailed(ctx context.Context, pkg *models.SLLMBenchmarkPackage, err error, cleanup bool) { + _ = pkg.SetStatus(ctx, task.UserCred, imageapi.IMAGE_STATUS_KILLED, err.Error()) + db.OpsLog.LogEvent(pkg, db.ACT_CREATE, err, task.UserCred) + logclient.AddActionLogWithStartable(task, pkg, logclient.ACT_CREATE, err, task.UserCred, false) + if cleanup { + if cleanupErr := pkg.StartDeleteTask(ctx, task.UserCred, nil, ""); cleanupErr != nil { + log.Errorf("cleanup benchmark package %s: %s", pkg.Id, cleanupErr) + } + } + task.SetStageFailed(ctx, jsonutils.NewString(err.Error())) +} + +func (task *LLMBenchmarkPackageImportTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + pkg := obj.(*models.SLLMBenchmarkPackage) + input := api.LLMBenchmarkPackageImportInput{} + if err := task.Params.Unmarshal(&input, "import_input"); err != nil { + task.taskFailed(ctx, pkg, err, false) + return + } + + task.SetStage("OnImportComplete", nil) + session := auth.GetAdminSession(ctx, options.Options.Region) + tmpDir, err := pkg.DoImport(ctx, task.UserCred, session, input.LLMBenchmarkPackageCreateInput) + if tmpDir != "" { + defer pkg.CleanupImportTmpDir(ctx, task.UserCred, tmpDir) + } + if err != nil { + task.OnImportCompleteFailed(ctx, pkg, jsonutils.NewString(err.Error())) + return + } + task.OnImportComplete(ctx, pkg, nil) +} + +func (task *LLMBenchmarkPackageImportTask) OnImportComplete(ctx context.Context, pkg *models.SLLMBenchmarkPackage, body jsonutils.JSONObject) { + input := api.LLMBenchmarkPackageImportInput{} + if err := task.Params.Unmarshal(&input, "import_input"); err != nil { + task.taskFailed(ctx, pkg, err, false) + return + } + + result := jsonutils.NewDict() + if input.BenchmarkSpec != nil { + benchmarkInput, err := models.PrepareLLMBenchmarkCreateInput(pkg, input.BenchmarkSpec) + if err != nil { + task.taskFailed(ctx, pkg, err, true) + return + } + benchmark, err := models.GetLLMBenchmarkManager().CreateAndStart(ctx, task.UserCred, pkg.GetOwnerId(), benchmarkInput) + if err != nil { + task.taskFailed(ctx, pkg, errors.Wrap(err, "create benchmark"), true) + return + } + result.Set("benchmark_id", jsonutils.NewString(benchmark.Id)) + } + + db.OpsLog.LogEvent(pkg, db.ACT_CREATE, pkg.GetShortDesc(ctx), task.UserCred) + logclient.AddActionLogWithStartable(task, pkg, logclient.ACT_CREATE, pkg.GetShortDesc(ctx), task.UserCred, true) + task.SetStageComplete(ctx, result) +} + +func (task *LLMBenchmarkPackageImportTask) OnImportCompleteFailed(ctx context.Context, pkg *models.SLLMBenchmarkPackage, body jsonutils.JSONObject) { + input := api.LLMBenchmarkPackageImportInput{} + _ = task.Params.Unmarshal(&input, "import_input") + task.taskFailed( + ctx, + pkg, + errors.Errorf("benchmark package import failed: %s", body), + input.BenchmarkSpec != nil, + ) +} diff --git a/pkg/llm/tasks/llm/llm_benchmark_run_task.go b/pkg/llm/tasks/llm/llm_benchmark_run_task.go new file mode 100644 index 0000000000..0bfce060f2 --- /dev/null +++ b/pkg/llm/tasks/llm/llm_benchmark_run_task.go @@ -0,0 +1,687 @@ +package llm + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + commonapi "yunion.io/x/onecloud/pkg/apis" + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + bench "yunion.io/x/onecloud/pkg/llm/benchmark" + "yunion.io/x/onecloud/pkg/llm/models" + "yunion.io/x/onecloud/pkg/llm/options" + llmutils "yunion.io/x/onecloud/pkg/llm/utils" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/auth" + computemod "yunion.io/x/onecloud/pkg/mcclient/modules/compute" +) + +type LLMBenchmarkRunTask struct { + taskman.STask +} + +const ( + preflightRemoteJSON = "/workdir/dataset-preflight.json" + preflightRemoteCSV = "/workdir/dataset-preflight.csv" + preflightRemoteLog = "/workdir/dataset-preflight.log" +) + +const ( + evaluationDatasetRemote = "/workdir/evaluation-dataset.jsonl" + evaluationDatasetLocal = "evaluation-dataset.jsonl" +) + +func init() { + taskman.RegisterTask(LLMBenchmarkRunTask{}) +} + +func applyBenchmarkRuntimeTokenizer(spec *bench.GuideLLMSpec, mount *models.LLMBenchmarkTokenizerMount) { + if spec == nil || mount == nil { + return + } + spec.Tokenizer = bench.GuideLLMLocalTokenizer(mount.ModelPath) +} + +func (task *LLMBenchmarkRunTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + b := obj.(*models.SLLMBenchmark) + if stoppedBenchmark(b) { + _, _ = b.FinishRun(ctx, nil) + task.SetStageComplete(ctx, nil) + return + } + _ = b.SetState(ctx, task.UserCred, api.LLMBenchmarkStateQueued, "") + task.SetStage("OnRunComplete", nil) + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + defer cleanupRunner(ctx, task.UserCred, b) + err := task.run(ctx, b) + persistErr := task.persistBenchmarkArtifacts(ctx, b) + if err == nil { + err = persistErr + } + return nil, err + }) +} + +func (task *LLMBenchmarkRunTask) OnRunComplete(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + b := obj.(*models.SLLMBenchmark) + if _, err := b.FinishRun(ctx, nil); err != nil { + task.SetStageFailed(ctx, jsonutils.NewString(err.Error())) + return + } + task.SetStageComplete(ctx, nil) +} + +func (task *LLMBenchmarkRunTask) OnRunCompleteFailed(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + b := obj.(*models.SLLMBenchmark) + reason, _ := body.GetString("__reason__") + if reason == "" { + reason = body.String() + } + state, err := b.FinishRun(ctx, errors.Error(reason)) + if err != nil { + task.SetStageFailed(ctx, jsonutils.NewString(err.Error())) + return + } + if state == api.LLMBenchmarkStateStopped { + task.SetStageComplete(ctx, nil) + return + } + task.SetStageFailed(ctx, jsonutils.NewString(reason)) +} + +func (task *LLMBenchmarkRunTask) run(ctx context.Context, b *models.SLLMBenchmark) error { + if err := os.MkdirAll(b.WorkDir, 0755); err != nil { + return errors.Wrap(err, "MkdirAll") + } + if err := os.WriteFile(filepath.Join(b.WorkDir, "target_snapshot.json"), []byte(b.TargetSnapshot), 0644); err != nil { + return errors.Wrap(err, "write target_snapshot") + } + llmObj, err := models.GetLLMManager().FetchById(b.LLMId) + if err != nil { + return errors.Wrap(err, "fetch LLM") + } + llm := llmObj.(*models.SLLM) + targetServer, err := llm.GetServer(ctx) + if err != nil { + return errors.Wrap(err, "GetServer") + } + imageObj, err := models.GetLLMImageManager().FetchById(b.LLMImageId) + if err != nil { + return errors.Wrap(err, "FetchById benchmark image") + } + image := imageObj.(*models.SLLMImage) + + var spec bench.GuideLLMSpec + if err := json.Unmarshal([]byte(b.GuideLLMSpec), &spec); err != nil { + return errors.Wrap(err, "unmarshal GuideLLMSpec") + } + var tokenizerMount *models.LLMBenchmarkTokenizerMount + if b.DatasetName == api.LLMBenchmarkDatasetSyntheticText { + tokenizerMount, err = b.ResolveTokenizerMount() + if err != nil { + return errors.Wrap(err, "resolve offline benchmark tokenizer") + } + applyBenchmarkRuntimeTokenizer(&spec, tokenizerMount) + } + runtimeSpec, err := json.Marshal(spec) + if err != nil { + return errors.Wrap(err, "marshal runtime GuideLLMSpec") + } + if err := os.WriteFile(filepath.Join(b.WorkDir, "spec.json"), runtimeSpec, 0644); err != nil { + return errors.Wrap(err, "write spec") + } + envs, err := bench.GuideLLMEnvs(spec) + if err != nil { + return errors.Wrap(err, "GuideLLMEnvs") + } + podInput, err := b.RunnerPodInput(image.ToContainerImage(), targetServer, tokenizerMount) + if err != nil { + return err + } + podInput.Pod.Containers[0].Envs = append(podInput.Pod.Containers[0].Envs, envMapToKVs(envs)...) + if stoppedBenchmark(b) { + return errors.Error("benchmark stopped") + } + + s := auth.GetSession(ctx, task.UserCred, options.Options.Region) + resp, err := computemod.Servers.Create(s, jsonutils.Marshal(podInput)) + if err != nil { + return errors.Wrap(err, "create runner pod") + } + runnerServerId, err := resp.GetString("id") + if err != nil { + return errors.Wrap(err, "runner server id") + } + _ = b.SetRunner(ctx, runnerServerId, "") + if stoppedBenchmark(b) { + return errors.Error("benchmark stopped") + } + + server, err := llmutils.WaitServerStatus(ctx, runnerServerId, []string{computeapi.VM_RUNNING}, 600) + if err != nil { + return errors.Wrap(err, "wait runner server") + } + if len(server.Containers) == 0 { + return errors.Error("runner pod has no containers") + } + containerId := server.Containers[0].Id + _ = b.SetRunner(ctx, runnerServerId, containerId) + if stoppedBenchmark(b) { + return errors.Error("benchmark stopped") + } + if b.DatasetName == api.LLMBenchmarkDatasetPackage { + if err := task.runDatasetPreflight(ctx, s, containerId, b, spec); err != nil { + return err + } + } + if stoppedBenchmark(b) { + return errors.Error("benchmark stopped") + } + if err := b.SetState(ctx, task.UserCred, api.LLMBenchmarkStateRunning, ""); err != nil { + return err + } + + phaseErr := executeGuideLLMPhase( + ctx, + s, + containerId, + b, + bench.GuideLLMRunCommand(), + "/workdir/benchmarks.json", + "/workdir/benchmarks.csv", + ) + if phaseErr != nil { + _ = copyBenchmarkArtifacts(s, containerId, b) + return phaseErr + } + if err := copyBenchmarkArtifacts(s, containerId, b); err != nil { + return err + } + + metrics, err := bench.ParseMetricsCSV(b.ResultCsv) + if err != nil { + _ = b.SetState(ctx, task.UserCred, api.LLMBenchmarkStateRunning, "parse metrics: "+err.Error()) + if b.DatasetName == api.LLMBenchmarkDatasetPackage { + return task.saveDatasetEvaluationFailure(ctx, b, "", 0, errors.Wrap(err, "parse formal metrics")) + } + return nil + } + if err := b.UpdateMetrics(ctx, task.UserCred, metrics); err != nil { + return err + } + if b.DatasetName == api.LLMBenchmarkDatasetPackage { + return task.runDatasetEvaluation(ctx, s, containerId, b, metrics.RequestTotal) + } + return nil +} + +func evaluationDatasetExportCommand(datasetPath string, requestTotal int) []string { + command := fmt.Sprintf( + "head -n %d %s > %s", + requestTotal, + shellPath(datasetPath), + evaluationDatasetRemote, + ) + return []string{"sh", "-lc", command} +} + +func datasetEvaluationFailure( + workDir, answerColumn string, + requestTotal int, + cause error, +) (bench.EvaluationResult, error) { + ret := bench.EvaluationResult{ + Summary: api.LLMBenchmarkDatasetEvaluation{ + State: api.LLMBenchmarkEvaluationStateError, + AnswerColumn: answerColumn, + RequestTotal: requestTotal, + Message: cause.Error(), + }, + LogPath: filepath.Join(workDir, "evaluation.log"), + } + err := os.WriteFile( + ret.LogPath, + []byte("evaluation error: "+cause.Error()+"\n"), + 0644, + ) + return ret, err +} + +func appendArtifactStorageFallback(logPath string, cause error) { + if logPath == "" || cause == nil { + return + } + file, err := os.OpenFile(logPath, os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + return + } + defer file.Close() + _, _ = fmt.Fprintf(file, "artifact storage fallback: %s\n", cause) +} + +func exportEvaluationDataset( + s *mcclient.ClientSession, + containerID, datasetPath string, + requestTotal int, +) error { + if requestTotal <= 0 { + return errors.Error("formal benchmark executed zero requests") + } + input := &computeapi.ContainerExecSyncInput{ + Command: evaluationDatasetExportCommand(datasetPath, requestTotal), + Timeout: 60, + } + response, err := computemod.Containers.PerformAction( + s, + containerID, + "exec-sync", + jsonutils.Marshal(input), + ) + if err != nil { + return errors.Wrap(err, "export evaluation dataset") + } + if code, _ := response.Int("exit_code"); code != 0 { + stderr, _ := response.GetString("stderr") + return fmt.Errorf("export evaluation dataset exited %d: %s", code, stderr) + } + return nil +} + +func (task *LLMBenchmarkRunTask) runDatasetEvaluation( + ctx context.Context, + s *mcclient.ClientSession, + containerID string, + b *models.SLLMBenchmark, + requestTotal int, +) error { + evaluating := api.LLMBenchmarkDatasetEvaluation{ + State: api.LLMBenchmarkEvaluationStateEvaluating, + RequestTotal: requestTotal, + } + if err := b.UpdateDatasetEvaluation(ctx, task.UserCred, evaluating, "", "", ""); err != nil { + return err + } + + pkgObj, err := models.GetLLMBenchmarkPackageManager().FetchById(b.BenchmarkPackageId) + if err != nil { + return task.saveDatasetEvaluationFailure(ctx, b, "", requestTotal, errors.Wrap(err, "fetch benchmark package")) + } + pkg := pkgObj.(*models.SLLMBenchmarkPackage) + if err := exportEvaluationDataset(s, containerID, b.DatasetPath, requestTotal); err != nil { + return task.saveDatasetEvaluationFailure(ctx, b, pkg.AnswerColumn, requestTotal, err) + } + localDataset := filepath.Join(b.WorkDir, evaluationDatasetLocal) + copied, err := copyArtifacts(s, containerID, map[string]string{ + evaluationDatasetRemote: localDataset, + }) + if err != nil || copied[evaluationDatasetRemote] == "" { + if err == nil { + err = errors.Error("evaluation dataset artifact not copied") + } + return task.saveDatasetEvaluationFailure(ctx, b, pkg.AnswerColumn, requestTotal, err) + } + defer os.Remove(localDataset) + + result, evaluationErr := bench.EvaluateDataset(ctx, bench.EvaluationInput{ + DatasetPath: localDataset, + BenchmarkPath: b.ResultJson, + AnswerColumn: pkg.AnswerColumn, + OutputDir: b.WorkDir, + }) + if evaluationErr != nil && result.Summary.State == "" { + return task.saveDatasetEvaluationFailure(ctx, b, pkg.AnswerColumn, requestTotal, evaluationErr) + } + if result.Summary.State != api.LLMBenchmarkEvaluationStateCompleted { + result.Summary.RequestTotal = requestTotal + } + if err := b.UpdateDatasetEvaluation( + ctx, + task.UserCred, + result.Summary, + result.ResultJSON, + result.ResultCSV, + result.LogPath, + ); err != nil { + return err + } + return nil +} + +func (task *LLMBenchmarkRunTask) saveDatasetEvaluationFailure( + ctx context.Context, + b *models.SLLMBenchmark, + answerColumn string, + requestTotal int, + cause error, +) error { + result, writeErr := datasetEvaluationFailure(b.WorkDir, answerColumn, requestTotal, cause) + if writeErr != nil { + result.Summary.Message += ": write evaluation log: " + writeErr.Error() + result.LogPath = "" + } + return b.UpdateDatasetEvaluation( + ctx, + task.UserCred, + result.Summary, + "", + "", + result.LogPath, + ) +} + +func (task *LLMBenchmarkRunTask) persistBenchmarkArtifacts( + ctx context.Context, + b *models.SLLMBenchmark, +) error { + local := b.ArtifactLocations() + store := bench.DefaultArtifactStore() + stored, storage, storageErr := store.Persist(ctx, b.ProjectId, b.Id, local) + message := "" + if storageErr != nil { + message = storageErr.Error() + appendArtifactStorageFallback(local["evaluation-log"], storageErr) + } + if err := b.UpdateArtifactLocations(ctx, task.UserCred, stored, storage, message); err != nil { + if storage == api.LLMBenchmarkArtifactStorageMinio { + _ = store.DeleteBenchmark(ctx, b.ProjectId, b.Id) + } + return err + } + if storage == api.LLMBenchmarkArtifactStorageMinio { + if err := store.RemoveLocal(local); err != nil { + log.Warningf("remove uploaded benchmark %s local artifacts: %s", b.Id, err) + } + } + return nil +} + +func evaluateDatasetPreflight(metrics *bench.LLMBenchmarkMetrics, phaseErr error) (api.LLMBenchmarkDatasetPreflight, error) { + ret := api.LLMBenchmarkDatasetPreflight{ + State: api.LLMBenchmarkStateCompleted, + ExpectedSamples: bench.DatasetPreflightSamples, + } + if metrics == nil { + ret.State = api.LLMBenchmarkStateError + if phaseErr == nil { + phaseErr = errors.Error("dataset preflight metrics not ready") + } + ret.Message = phaseErr.Error() + return ret, phaseErr + } + ret.ActualSamples = metrics.RequestTotal + ret.Successful = metrics.RequestSuccessful + ret.Errored = metrics.RequestErrored + if metrics.ErrorRate != nil { + ret.ErrorRate = *metrics.ErrorRate + } else if metrics.RequestTotal > 0 { + ret.ErrorRate = float64(metrics.RequestErrored) / float64(metrics.RequestTotal) + } + if metrics.RequestLatencyMeanSec != nil { + ret.LatencyMeanSeconds = *metrics.RequestLatencyMeanSec + } + if metrics.RequestTotal == 0 { + ret.State = api.LLMBenchmarkStateError + ret.Message = "dataset preflight executed zero requests" + return ret, errors.Error(ret.Message) + } + if metrics.RequestSuccessful == 0 { + ret.State = api.LLMBenchmarkStateError + ret.Message = fmt.Sprintf("dataset preflight failed all %d requests", metrics.RequestTotal) + if phaseErr != nil { + ret.Message += ": " + phaseErr.Error() + } + return ret, errors.Error(ret.Message) + } + if metrics.RequestErrored > 0 || phaseErr != nil { + ret.Message = fmt.Sprintf("dataset preflight completed with %d successful and %d errored requests", metrics.RequestSuccessful, metrics.RequestErrored) + if phaseErr != nil { + ret.Message += ": " + phaseErr.Error() + } + } + return ret, nil +} + +func (task *LLMBenchmarkRunTask) runDatasetPreflight( + ctx context.Context, + s *mcclient.ClientSession, + containerId string, + b *models.SLLMBenchmark, + formal bench.GuideLLMSpec, +) error { + preflight, err := bench.BuildGuideLLMPreflightSpec(formal, b.MaxDurationSeconds) + if err != nil { + return err + } + command, err := bench.GuideLLMPreflightRunCommand(preflight) + if err != nil { + return err + } + if err := b.SetState(ctx, task.UserCred, api.LLMBenchmarkStateValidating, ""); err != nil { + return err + } + validating := api.LLMBenchmarkDatasetPreflight{ + State: api.LLMBenchmarkStateValidating, + ExpectedSamples: bench.DatasetPreflightSamples, + } + if err := b.UpdateDatasetPreflight(ctx, task.UserCred, validating, "", ""); err != nil { + return errors.Wrap(err, "start dataset preflight") + } + + phaseErr := executeGuideLLMPhase(ctx, s, containerId, b, command, preflightRemoteJSON, preflightRemoteCSV) + paths := map[string]string{ + preflightRemoteJSON: filepath.Join(b.WorkDir, "dataset-preflight.json"), + preflightRemoteCSV: filepath.Join(b.WorkDir, "dataset-preflight.csv"), + preflightRemoteLog: filepath.Join(b.WorkDir, "dataset-preflight.log"), + } + copied, copyErr := copyArtifacts(s, containerId, paths) + if phaseErr == nil { + phaseErr = copyErr + } + + if stoppedBenchmark(b) { + summary := api.LLMBenchmarkDatasetPreflight{ + State: api.LLMBenchmarkStateStopped, + ExpectedSamples: bench.DatasetPreflightSamples, + Message: "benchmark stopped during dataset preflight", + } + _ = b.UpdateDatasetPreflight(ctx, task.UserCred, summary, copied[preflightRemoteJSON], copied[preflightRemoteLog]) + return errors.Error(summary.Message) + } + + var metrics *bench.LLMBenchmarkMetrics + if csvPath := copied[preflightRemoteCSV]; csvPath != "" { + metrics, err = bench.ParseMetricsCSV(csvPath) + if phaseErr == nil { + phaseErr = err + } + } + if metrics == nil || metrics.RequestSuccessful == 0 { + phaseErr = datasetPreflightPhaseError(phaseErr, copied[preflightRemoteLog]) + } + summary, decisionErr := evaluateDatasetPreflight(metrics, phaseErr) + if err := b.UpdateDatasetPreflight(ctx, task.UserCred, summary, copied[preflightRemoteJSON], copied[preflightRemoteLog]); err != nil { + return errors.Wrap(err, "save dataset preflight") + } + return decisionErr +} + +func datasetPreflightPhaseError(phaseErr error, logPath string) error { + f, err := os.Open(logPath) + if err != nil { + return phaseErr + } + defer f.Close() + + scanner := bufio.NewScanner(f) + scanner.Buffer(make([]byte, 64*1024), 1024*1024) + last := "" + for scanner.Scan() { + if line := strings.TrimSpace(scanner.Text()); line != "" { + last = line + } + } + if last == "" { + return phaseErr + } + if phaseErr == nil { + return errors.Error(last) + } + if strings.Contains(phaseErr.Error(), last) { + return phaseErr + } + return errors.Wrap(phaseErr, last) +} + +func executeGuideLLMPhase( + ctx context.Context, + s *mcclient.ClientSession, + containerId string, + b *models.SLLMBenchmark, + command []string, + jsonPath, csvPath string, +) error { + execInput := &computeapi.ContainerExecSyncInput{ + Command: command, + Timeout: int64(b.MaxDurationSeconds + 120), + } + execResp, err := computemod.Containers.PerformAction(s, containerId, "exec-sync", jsonutils.Marshal(execInput)) + if err != nil { + if waitBenchmarkResults(ctx, s, containerId, b, jsonPath, csvPath, b.MaxDurationSeconds+30) != nil { + return errors.Wrap(err, "guidellm exec") + } + return nil + } + if code, _ := execResp.Int("exit_code"); code != 0 { + stderr, _ := execResp.GetString("stderr") + return fmt.Errorf("guidellm exited %d: %s", code, stderr) + } + return nil +} + +func waitBenchmarkResults(ctx context.Context, s *mcclient.ClientSession, containerId string, b *models.SLLMBenchmark, jsonPath, csvPath string, seconds int) error { + if seconds <= 0 { + seconds = 30 + } + deadline := time.Now().Add(time.Duration(seconds) * time.Second) + for time.Now().Before(deadline) { + if stoppedBenchmark(b) { + return errors.Error("benchmark stopped") + } + if benchmarkResultsReady(s, containerId, jsonPath, csvPath) { + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(2 * time.Second): + } + } + return errors.Error("benchmark result artifacts not ready") +} + +func benchmarkResultsReady(s *mcclient.ClientSession, containerId, jsonPath, csvPath string) bool { + command := fmt.Sprintf("test -s %s -a -s %s", shellPath(jsonPath), shellPath(csvPath)) + input := &computeapi.ContainerExecSyncInput{ + Command: []string{"sh", "-lc", command}, + Timeout: 10, + } + resp, err := computemod.Containers.PerformAction(s, containerId, "exec-sync", jsonutils.Marshal(input)) + if err != nil { + return false + } + code, _ := resp.Int("exit_code") + return code == 0 +} + +func shellPath(path string) string { + return "'" + strings.ReplaceAll(path, "'", "'\"'\"'") + "'" +} + +func copyBenchmarkArtifacts(s *mcclient.ClientSession, containerId string, b *models.SLLMBenchmark) error { + paths := map[string]string{ + "/workdir/guidellm.log": filepath.Join(b.WorkDir, "guidellm.log"), + "/workdir/benchmarks.json": filepath.Join(b.WorkDir, "benchmarks.json"), + "/workdir/benchmarks.csv": filepath.Join(b.WorkDir, "benchmarks.csv"), + } + copied, firstErr := copyArtifacts(s, containerId, paths) + _, err := db.Update(b, func() error { + applyBenchmarkArtifactPaths(b, copied) + return nil + }) + if err != nil { + return err + } + return firstErr +} + +func applyBenchmarkArtifactPaths(b *models.SLLMBenchmark, copied map[string]string) { + b.LogPath = copied["/workdir/guidellm.log"] + b.ResultJson = copied["/workdir/benchmarks.json"] + b.ResultCsv = copied["/workdir/benchmarks.csv"] +} + +func copyArtifacts(s *mcclient.ClientSession, containerId string, paths map[string]string) (map[string]string, error) { + copied := map[string]string{} + var firstErr error + for remote, local := range paths { + out, err := os.Create(local) + if err != nil { + if firstErr == nil { + firstErr = errors.Wrapf(err, "create %s", local) + } + continue + } + err = computemod.Containers.CopyFrom(s, containerId, remote, out) + closeErr := out.Close() + if err != nil { + if firstErr == nil { + firstErr = errors.Wrapf(err, "copy %s", remote) + } + continue + } + if closeErr != nil { + if firstErr == nil { + firstErr = errors.Wrap(closeErr, "close artifact") + } + continue + } + copied[remote] = local + } + return copied, firstErr +} + +func cleanupRunner(ctx context.Context, userCred mcclient.TokenCredential, b *models.SLLMBenchmark) { + if b.RunnerServerId == "" { + return + } + _ = b.DeleteRunnerServer(ctx, userCred) +} + +func stoppedBenchmark(b *models.SLLMBenchmark) bool { + obj, err := models.GetLLMBenchmarkManager().FetchById(b.Id) + if err != nil { + return b.StopRequested + } + cur := obj.(*models.SLLMBenchmark) + return cur.StopRequested || cur.State == api.LLMBenchmarkStateStopped +} + +func envMapToKVs(envs map[string]string) []*commonapi.ContainerKeyValue { + ret := make([]*commonapi.ContainerKeyValue, 0, len(envs)) + for k, v := range envs { + ret = append(ret, &commonapi.ContainerKeyValue{Key: k, Value: v}) + } + return ret +} diff --git a/pkg/mcclient/modules/llm/mod_llm_benchmark.go b/pkg/mcclient/modules/llm/mod_llm_benchmark.go new file mode 100644 index 0000000000..aad7f2c6f3 --- /dev/null +++ b/pkg/mcclient/modules/llm/mod_llm_benchmark.go @@ -0,0 +1,60 @@ +package llm + +import ( + "fmt" + "io" + "net/url" + "strconv" + + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modulebase" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +type LLMBenchmarkManager struct { + modulebase.ResourceManager +} + +var ( + LLMBenchmarks LLMBenchmarkManager +) + +func (m *LLMBenchmarkManager) Artifact(s *mcclient.ClientSession, id string, kind string) (io.ReadCloser, int64, error) { + path := fmt.Sprintf("/%s/%s/artifacts/%s", m.URLPath(), url.PathEscape(id), url.PathEscape(kind)) + resp, err := modulebase.RawRequest(m.ResourceManager, s, "GET", path, nil, nil) + if err == nil && resp.StatusCode >= 200 && resp.StatusCode < 300 { + sizeBytes, err := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64) + if err != nil { + log.Errorf("download benchmark artifact unknown size") + sizeBytes = -1 + } + return resp.Body, sizeBytes, nil + } + _, _, err = s.ParseJSONResponse("", resp, err) + return nil, -1, err +} + +func init() { + LLMBenchmarks = LLMBenchmarkManager{ + ResourceManager: modules.NewLLMManager("llm_benchmark", "llm_benchmarks", + []string{ + "ID", + "Name", + "LLMDeploymentId", + "LLMDeployment", + "LLMId", + "State", + "RequestRate", + "TotalRequests", + "RequestTotal", + "RequestSuccessful", + "RequestErrored", + "ErrorRate", + }, + []string{}, + ), + } + modules.Register(&LLMBenchmarks) +} diff --git a/pkg/mcclient/modules/llm/mod_llm_benchmark_package.go b/pkg/mcclient/modules/llm/mod_llm_benchmark_package.go new file mode 100644 index 0000000000..2d60774604 --- /dev/null +++ b/pkg/mcclient/modules/llm/mod_llm_benchmark_package.go @@ -0,0 +1,35 @@ +package llm + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modulebase" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +type LLMBenchmarkPackageManager struct { + modulebase.ResourceManager +} + +var ( + LLMBenchmarkPackages LLMBenchmarkPackageManager +) + +func init() { + LLMBenchmarkPackages = LLMBenchmarkPackageManager{ + ResourceManager: modules.NewLLMManager("llm_benchmark_package", "llm_benchmark_packages", + []string{ + "ID", + "Name", + "Status", + "Source", + "RepoId", + "Revision", + "FilePath", + "Format", + "ImageId", + "DatasetPath", + }, + []string{}, + ), + } + modules.Register(&LLMBenchmarkPackages) +} diff --git a/pkg/mcclient/options/llm/image.go b/pkg/mcclient/options/llm/image.go index 4670730338..4f619554e0 100644 --- a/pkg/mcclient/options/llm/image.go +++ b/pkg/mcclient/options/llm/image.go @@ -18,7 +18,7 @@ func (o *LLMImageShowOptions) Params() (jsonutils.JSONObject, error) { type LLMImageListOptions struct { options.BaseListOptions - LLMType string `json:"llm_type" choices:"ollama|dify|comfyui|vllm|sglang|hermes-agent|llm-router|desktop" help:"filter by llm type"` + LLMType string `json:"llm_type" choices:"ollama|dify|comfyui|vllm|sglang|hermes-agent|llm-router|desktop|benchmark" help:"filter by llm type"` } func (o *LLMImageListOptions) Params() (jsonutils.JSONObject, error) { @@ -30,7 +30,7 @@ type LLMImageCreateOptions struct { IMAGE_NAME string `json:"image_name"` IMAGE_LABEL string `json:"image_label"` CredentialId string `json:"credential_id"` - LLM_TYPE string `json:"llm_type" choices:"ollama|dify|comfyui|vllm|sglang|hermes-agent|llm-router|desktop" help:"llm type: ollama, comfyui, vllm, sglang, hermes-agent, llm-router, desktop or dify"` + LLM_TYPE string `json:"llm_type" choices:"ollama|dify|comfyui|vllm|sglang|hermes-agent|llm-router|desktop|benchmark" help:"llm type: ollama, comfyui, vllm, sglang, hermes-agent, llm-router, desktop, benchmark or dify"` } func (o *LLMImageCreateOptions) Params() (jsonutils.JSONObject, error) { @@ -44,7 +44,7 @@ type LLMImageUpdateOptions struct { ImageName string `json:"image_name"` ImageLabel string `json:"image_label"` CredentialId string `json:"credential_id"` - LlmType string `json:"llm_type" choices:"ollama|dify|vllm|sglang|comfyui|hermes-agent|llm-router|desktop" help:"llm type: ollama, comfyui, vllm, sglang, hermes-agent, llm-router, desktop or dify"` + LlmType string `json:"llm_type" choices:"ollama|dify|vllm|sglang|comfyui|hermes-agent|llm-router|desktop|benchmark" help:"llm type: ollama, comfyui, vllm, sglang, hermes-agent, llm-router, desktop, benchmark or dify"` AppName string `json:"app_name" help:"desktop application identifier (LinuxServer image id), e.g. firefox, chromium, steam, webtop-ubuntu-xfce"` } diff --git a/pkg/mcclient/options/llm/llm_benchmark.go b/pkg/mcclient/options/llm/llm_benchmark.go new file mode 100644 index 0000000000..065f325b3c --- /dev/null +++ b/pkg/mcclient/options/llm/llm_benchmark.go @@ -0,0 +1,134 @@ +package llm + +import ( + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/apis" + "yunion.io/x/onecloud/pkg/mcclient/options" +) + +type LLMBenchmarkShowOptions struct { + options.BaseShowOptions +} + +func (o *LLMBenchmarkShowOptions) Params() (jsonutils.JSONObject, error) { + return options.ListStructToParams(o) +} + +type LLMBenchmarkListOptions struct { + options.BaseListOptions + + LLMId string `json:"llm_id" token:"llm" help:"filter by LLM id"` + LLMDeploymentId string `json:"llm_deployment_id" token:"llm-deployment" help:"filter by LLM deployment id or name"` + State string `json:"state" choices:"pending|queued|running|completed|stopped|error" help:"filter by state"` +} + +func (o *LLMBenchmarkListOptions) Params() (jsonutils.JSONObject, error) { + return options.ListStructToParams(o) +} + +type LLMBenchmarkCreateOptions struct { + apis.VirtualResourceCreateInput + + LLMId string `json:"llm_id" token:"llm" help:"target LLM id or name"` + LLMDeploymentId string `json:"llm_deployment_id" token:"llm-deployment" help:"target LLM deployment id or name"` + BenchmarkImage string `json:"benchmark_image" help:"benchmark llm_image id or name"` + BenchmarkPackage string `json:"benchmark_package" help:"benchmark package id or name"` + + RequestFormat string `json:"request_format" default:"/v1/chat/completions"` + Model string `json:"model"` + Profile string `json:"profile" choices:"constant"` + RequestRate int `json:"request_rate"` + + TotalRequests int `json:"total_requests"` + MaxDurationSeconds int `json:"max_duration_seconds"` + MaxErrors int `json:"max_errors"` + + DatasetName string `json:"dataset_name" choices:"synthetic_text|benchmark_package"` + DatasetInputTokens int `json:"dataset_input_tokens"` + DatasetOutputTokens int `json:"dataset_output_tokens"` + DatasetPath string `json:"dataset_path"` +} + +func (o *LLMBenchmarkCreateOptions) Params() (jsonutils.JSONObject, error) { + return jsonutils.Marshal(o), nil +} + +type LLMBenchmarkUpdateOptions struct { + options.BaseIdOptions + + Name string `json:"name,omitempty"` + Description *string `json:"description,omitempty"` + Model *string `json:"model,omitempty"` + RequestRate *int `json:"request_rate,omitempty"` + TotalRequests *int `json:"total_requests,omitempty"` + MaxDurationSeconds *int `json:"max_duration_seconds,omitempty"` + MaxErrors *int `json:"max_errors,omitempty"` + DatasetInputTokens *int `json:"dataset_input_tokens,omitempty"` + DatasetOutputTokens *int `json:"dataset_output_tokens,omitempty"` +} + +func (o *LLMBenchmarkUpdateOptions) GetId() string { + return o.ID +} + +func (o *LLMBenchmarkUpdateOptions) Params() (jsonutils.JSONObject, error) { + return options.StructToParams(o) +} + +type LLMBenchmarkCopyOptions struct { + options.BaseIdOptions + + NAME string `json:"name"` + LLMDeploymentId string `json:"llm_deployment_id"` + Description *string `json:"description,omitempty"` + Model *string `json:"model,omitempty"` + RequestRate *int `json:"request_rate,omitempty"` + TotalRequests *int `json:"total_requests,omitempty"` + MaxDurationSeconds *int `json:"max_duration_seconds,omitempty"` + MaxErrors *int `json:"max_errors,omitempty"` + DatasetInputTokens *int `json:"dataset_input_tokens,omitempty"` + DatasetOutputTokens *int `json:"dataset_output_tokens,omitempty"` +} + +func (o *LLMBenchmarkCopyOptions) GetId() string { + return o.ID +} + +func (o *LLMBenchmarkCopyOptions) Params() (jsonutils.JSONObject, error) { + return options.StructToParams(o) +} + +type LLMBenchmarkRetestOptions struct { + options.BaseIdOptions +} + +func (o *LLMBenchmarkRetestOptions) GetId() string { + return o.ID +} + +func (o *LLMBenchmarkRetestOptions) Params() (jsonutils.JSONObject, error) { + return jsonutils.NewDict(), nil +} + +type LLMBenchmarkDeleteOptions struct { + options.BaseIdOptions +} + +func (o *LLMBenchmarkDeleteOptions) GetId() string { + return o.ID +} + +type LLMBenchmarkStopOptions struct { + options.BaseIdOptions +} + +func (o *LLMBenchmarkStopOptions) GetId() string { + return o.ID +} + +type LLMBenchmarkArtifactOptions struct { + ID string `help:"ID or name of benchmark"` + Type string `help:"artifact type" choices:"log|json|csv"` + Output string `short-token:"o" help:"destination file, if omitted, output to stdout"` +} diff --git a/pkg/mcclient/options/llm/llm_benchmark_package.go b/pkg/mcclient/options/llm/llm_benchmark_package.go new file mode 100644 index 0000000000..14d02e9d4e --- /dev/null +++ b/pkg/mcclient/options/llm/llm_benchmark_package.go @@ -0,0 +1,73 @@ +package llm + +import ( + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/apis" + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/mcclient/options" +) + +type LLMBenchmarkPackageShowOptions struct { + options.BaseShowOptions +} + +func (o *LLMBenchmarkPackageShowOptions) Params() (jsonutils.JSONObject, error) { + return options.StructToParams(o) +} + +type LLMBenchmarkPackageListOptions struct { + options.BaseListOptions + + Source string `json:"source" choices:"huggingface"` + RepoId string `json:"repo_id"` + Format string `json:"format" choices:"guidellm_jsonl"` +} + +func (o *LLMBenchmarkPackageListOptions) Params() (jsonutils.JSONObject, error) { + return options.ListStructToParams(o) +} + +type LLMBenchmarkPackageCreateOptions struct { + apis.SharableVirtualResourceCreateInput + + Source string `json:"source" choices:"huggingface"` + RepoId string `json:"repo_id"` + Revision string `json:"revision"` + FilePath string `json:"file_path"` + Format string `json:"format" choices:"guidellm_jsonl"` + ImageId string `json:"image_id"` +} + +func (o *LLMBenchmarkPackageCreateOptions) Params() (jsonutils.JSONObject, error) { + return jsonutils.Marshal(o), nil +} + +type LLMBenchmarkPackageImportOptions struct { + NAME string `json:"name" help:"benchmark package name"` + REPO_ID string `json:"repo_id" help:"HuggingFace dataset repo id, e.g. org/dataset"` + FILE_PATH string `json:"file_path" help:"JSONL file path in the dataset repo"` + Revision string `json:"revision" help:"HuggingFace revision, default main"` +} + +func (o *LLMBenchmarkPackageImportOptions) Params() (jsonutils.JSONObject, error) { + input := api.LLMBenchmarkPackageImportInput{ + LLMBenchmarkPackageCreateInput: api.LLMBenchmarkPackageCreateInput{ + Source: api.LLMBenchmarkPackageSourceHuggingFace, + RepoId: o.REPO_ID, + Revision: o.Revision, + FilePath: o.FILE_PATH, + Format: api.LLMBenchmarkPackageFormatGuideLLMJSONL, + }, + } + input.Name = o.NAME + return jsonutils.Marshal(input), nil +} + +type LLMBenchmarkPackageDeleteOptions struct { + options.BaseIdOptions +} + +func (o *LLMBenchmarkPackageDeleteOptions) GetId() string { + return o.ID +}