From 2bbd1f792c5941e32bd788f945af396d4f3360bf Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Fri, 24 Oct 2025 13:29:40 +0800 Subject: [PATCH] Automated cherry pick of #23282: [OSPP]feature: Auto deploy llm and dify (#23601) * feat(llm): add llm-create * fix(llm): fix llm-create, add llm-list & llm-show * fix(llm): format file names * fear(llm): add exec stream * fear(llm): init llm model cache * fix(llm): move llm model cache to LocalImageCache * feat(llm): auto clean model cache * fix(llm): auto clean tmp model cache * feat(dify): init dify-create * fix(dify): change dify default registry * feat: support create model with gguf file * feat(llm): support more modelfile option for gguf file * feat(dify): init user customize dify parameters * feat: update llm as a service in climc * fix: delete llm from region service * feat: init llm service (with error) * fix: resolve errors (task cant callback yet) * feat: add PerformRequestHostActionByOtherService for container * fix: make llm service usable * feat: delete llm and dify after guest deleted * feat(llm): add llm-image * feat(llm): add llm-model * feat: mv old llm to ollama * feat(llm): init llm-batch-create(can not run) * fix(llm): make llm-create usable * fix(llm): add pull-model step * feat(llm): add list and delete * fix(llm): ollama pull official model don't rely on host & region's code any more * feat(llm): add dify-model * fix: remove llm's code in host & compute * fix: remove remain code in compute * feat(llm): Abstract out and reuse the llm_model logic * feat(llm): abstract llm_base from llm * fix(llm): make dify usable * feat(llm): add sync_dify_images.sh * fix(llm): format import * feat(llm): support start & stop for llm and dify * feat(llm): add model-update * fix(llm): gendocgo --------- Co-authored-by: cwz --- build/docker/Dockerfile.llm | 3 + cmd/climc/main.go | 1 + cmd/climc/shell/llm/dify.go | 17 + cmd/climc/shell/llm/dify_model.go | 20 + cmd/climc/shell/llm/image.go | 19 + cmd/climc/shell/llm/llm.go | 18 + cmd/climc/shell/llm/llm_model.go | 20 + cmd/llm/main.go | 12 + pkg/apis/const.go | 2 + pkg/apis/llm/dify.go | 24 + pkg/apis/llm/dify_consts.go | 195 ++++++++ pkg/apis/llm/dify_files.go | 290 ++++++++++++ pkg/apis/llm/doc.go | 15 + pkg/apis/llm/image.go | 26 ++ pkg/apis/llm/llm.go | 45 ++ pkg/apis/llm/llm_const.go | 39 ++ pkg/apis/llm/llm_container.go | 30 ++ pkg/apis/llm/model.go | 211 +++++++++ pkg/apis/llm/ollama.go | 111 +++++ pkg/apis/llm/ollama_const.go | 52 +++ pkg/apis/llm/volume.go | 158 +++++++ pkg/llm/drivers/llm_container/doc.go | 1 + pkg/llm/drivers/llm_container/ollama.go | 237 ++++++++++ pkg/llm/models/access_info.go | 48 ++ pkg/llm/models/dify.go | 292 ++++++++++++ pkg/llm/models/dify_container_shared_envs.go | 242 ++++++++++ pkg/llm/models/dify_containers.go | 442 ++++++++++++++++++ pkg/llm/models/dify_model.go | 125 +++++ pkg/llm/models/dify_pod.go | 130 ++++++ pkg/llm/models/doc.go | 15 + pkg/llm/models/image.go | 153 ++++++ pkg/llm/models/initdb.go | 25 + pkg/llm/models/llm.go | 321 +++++++++++++ pkg/llm/models/llm_base.go | 307 ++++++++++++ pkg/llm/models/llm_container.go | 91 ++++ pkg/llm/models/llm_container_driver.go | 90 ++++ pkg/llm/models/llm_model.go | 199 ++++++++ pkg/llm/models/llm_pod.go | 172 +++++++ pkg/llm/models/model.go | 115 +++++ pkg/llm/models/volume.go | 67 +++ pkg/llm/options/doc.go | 15 + pkg/llm/options/option.go | 28 ++ pkg/llm/service/doc.go | 1 + pkg/llm/service/handler.go | 47 ++ pkg/llm/service/service.go | 47 ++ pkg/llm/tasks/dify/dify_batch_create_task.go | 40 ++ pkg/llm/tasks/dify/dify_create_task.go | 154 ++++++ pkg/llm/tasks/dify/dify_delete_task.go | 87 ++++ pkg/llm/tasks/dify/dify_start_task.go | 80 ++++ pkg/llm/tasks/dify/dify_stop_task.go | 96 ++++ pkg/llm/tasks/dify/doc.go | 1 + pkg/llm/tasks/doc.go | 15 + pkg/llm/tasks/llm/doc.go | 1 + pkg/llm/tasks/llm/llm_batch_create_task.go | 40 ++ pkg/llm/tasks/llm/llm_create_task.go | 172 +++++++ pkg/llm/tasks/llm/llm_delete_task.go | 108 +++++ .../tasks/llm/llm_install_gguf_file_task.go | 85 ++++ pkg/llm/tasks/llm/llm_pull_model_task.go | 74 +++ pkg/llm/tasks/llm/llm_start_task.go | 80 ++++ pkg/llm/tasks/llm/llm_stop_task.go | 96 ++++ pkg/llm/tasks/llm_container/doc.go | 1 + .../llm_container_delete_task.go | 37 ++ pkg/llm/tasks/loader.go | 22 + pkg/llm/tasks/volume/doc.go | 1 + pkg/llm/tasks/volume/volume_delete_task.go | 70 +++ pkg/llm/tasks/worker/doc.go | 1 + pkg/llm/tasks/worker/startworker.go | 38 ++ pkg/llm/utils/cloudutil.go | 173 +++++++ pkg/llm/utils/doc.go | 15 + pkg/mcclient/modules/llm/doc.go | 15 + pkg/mcclient/modules/llm/mod_dify.go | 23 + pkg/mcclient/modules/llm/mod_dify_model.go | 24 + pkg/mcclient/modules/llm/mod_image.go | 24 + pkg/mcclient/modules/llm/mod_llm.go | 23 + pkg/mcclient/modules/llm/mod_llm_model.go | 24 + pkg/mcclient/modules/llm/mod_task.go | 27 ++ pkg/mcclient/modules/managers.go | 6 + pkg/mcclient/options/llm/dify.go | 74 +++ pkg/mcclient/options/llm/dify_model.go | 78 ++++ pkg/mcclient/options/llm/doc.go | 15 + pkg/mcclient/options/llm/image.go | 60 +++ pkg/mcclient/options/llm/llm.go | 99 ++++ pkg/mcclient/options/llm/llm_model.go | 74 +++ pkg/mcclient/options/llm/llm_model_base.go | 220 +++++++++ scripts/sync_dify_images.sh | 59 +++ 85 files changed, 6850 insertions(+) create mode 100644 build/docker/Dockerfile.llm create mode 100644 cmd/climc/shell/llm/dify.go create mode 100644 cmd/climc/shell/llm/dify_model.go create mode 100644 cmd/climc/shell/llm/image.go create mode 100644 cmd/climc/shell/llm/llm.go create mode 100644 cmd/climc/shell/llm/llm_model.go create mode 100644 cmd/llm/main.go create mode 100644 pkg/apis/llm/dify.go create mode 100644 pkg/apis/llm/dify_consts.go create mode 100644 pkg/apis/llm/dify_files.go create mode 100644 pkg/apis/llm/doc.go create mode 100644 pkg/apis/llm/image.go create mode 100644 pkg/apis/llm/llm.go create mode 100644 pkg/apis/llm/llm_const.go create mode 100644 pkg/apis/llm/llm_container.go create mode 100644 pkg/apis/llm/model.go create mode 100644 pkg/apis/llm/ollama.go create mode 100644 pkg/apis/llm/ollama_const.go create mode 100644 pkg/apis/llm/volume.go create mode 100644 pkg/llm/drivers/llm_container/doc.go create mode 100644 pkg/llm/drivers/llm_container/ollama.go create mode 100644 pkg/llm/models/access_info.go create mode 100644 pkg/llm/models/dify.go create mode 100644 pkg/llm/models/dify_container_shared_envs.go create mode 100644 pkg/llm/models/dify_containers.go create mode 100644 pkg/llm/models/dify_model.go create mode 100644 pkg/llm/models/dify_pod.go create mode 100644 pkg/llm/models/doc.go create mode 100644 pkg/llm/models/image.go create mode 100644 pkg/llm/models/initdb.go create mode 100644 pkg/llm/models/llm.go create mode 100644 pkg/llm/models/llm_base.go create mode 100644 pkg/llm/models/llm_container.go create mode 100644 pkg/llm/models/llm_container_driver.go create mode 100644 pkg/llm/models/llm_model.go create mode 100644 pkg/llm/models/llm_pod.go create mode 100644 pkg/llm/models/model.go create mode 100644 pkg/llm/models/volume.go create mode 100644 pkg/llm/options/doc.go create mode 100644 pkg/llm/options/option.go create mode 100644 pkg/llm/service/doc.go create mode 100644 pkg/llm/service/handler.go create mode 100644 pkg/llm/service/service.go create mode 100644 pkg/llm/tasks/dify/dify_batch_create_task.go create mode 100644 pkg/llm/tasks/dify/dify_create_task.go create mode 100644 pkg/llm/tasks/dify/dify_delete_task.go create mode 100644 pkg/llm/tasks/dify/dify_start_task.go create mode 100644 pkg/llm/tasks/dify/dify_stop_task.go create mode 100644 pkg/llm/tasks/dify/doc.go create mode 100644 pkg/llm/tasks/doc.go create mode 100644 pkg/llm/tasks/llm/doc.go create mode 100644 pkg/llm/tasks/llm/llm_batch_create_task.go create mode 100644 pkg/llm/tasks/llm/llm_create_task.go create mode 100644 pkg/llm/tasks/llm/llm_delete_task.go create mode 100644 pkg/llm/tasks/llm/llm_install_gguf_file_task.go create mode 100644 pkg/llm/tasks/llm/llm_pull_model_task.go create mode 100644 pkg/llm/tasks/llm/llm_start_task.go create mode 100644 pkg/llm/tasks/llm/llm_stop_task.go create mode 100644 pkg/llm/tasks/llm_container/doc.go create mode 100644 pkg/llm/tasks/llm_container/llm_container_delete_task.go create mode 100644 pkg/llm/tasks/loader.go create mode 100644 pkg/llm/tasks/volume/doc.go create mode 100644 pkg/llm/tasks/volume/volume_delete_task.go create mode 100644 pkg/llm/tasks/worker/doc.go create mode 100644 pkg/llm/tasks/worker/startworker.go create mode 100644 pkg/llm/utils/cloudutil.go create mode 100644 pkg/llm/utils/doc.go create mode 100644 pkg/mcclient/modules/llm/doc.go create mode 100644 pkg/mcclient/modules/llm/mod_dify.go create mode 100644 pkg/mcclient/modules/llm/mod_dify_model.go create mode 100644 pkg/mcclient/modules/llm/mod_image.go create mode 100644 pkg/mcclient/modules/llm/mod_llm.go create mode 100644 pkg/mcclient/modules/llm/mod_llm_model.go create mode 100644 pkg/mcclient/modules/llm/mod_task.go create mode 100644 pkg/mcclient/options/llm/dify.go create mode 100644 pkg/mcclient/options/llm/dify_model.go create mode 100644 pkg/mcclient/options/llm/doc.go create mode 100644 pkg/mcclient/options/llm/image.go create mode 100644 pkg/mcclient/options/llm/llm.go create mode 100644 pkg/mcclient/options/llm/llm_model.go create mode 100644 pkg/mcclient/options/llm/llm_model_base.go create mode 100644 scripts/sync_dify_images.sh diff --git a/build/docker/Dockerfile.llm b/build/docker/Dockerfile.llm new file mode 100644 index 0000000000..1c1009db68 --- /dev/null +++ b/build/docker/Dockerfile.llm @@ -0,0 +1,3 @@ +FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5-1 + +ADD ./_output/alpine-build/bin/llm /opt/yunion/bin/llm diff --git a/cmd/climc/main.go b/cmd/climc/main.go index f10a5671bc..3933024778 100644 --- a/cmd/climc/main.go +++ b/cmd/climc/main.go @@ -30,6 +30,7 @@ import ( _ "yunion.io/x/onecloud/cmd/climc/shell/identity" _ "yunion.io/x/onecloud/cmd/climc/shell/image" _ "yunion.io/x/onecloud/cmd/climc/shell/k8s" + _ "yunion.io/x/onecloud/cmd/climc/shell/llm" _ "yunion.io/x/onecloud/cmd/climc/shell/logger" _ "yunion.io/x/onecloud/cmd/climc/shell/misc" _ "yunion.io/x/onecloud/cmd/climc/shell/monitor" diff --git a/cmd/climc/shell/llm/dify.go b/cmd/climc/shell/llm/dify.go new file mode 100644 index 0000000000..e1fc62431e --- /dev/null +++ b/cmd/climc/shell/llm/dify.go @@ -0,0 +1,17 @@ +package llm + +import ( + "yunion.io/x/onecloud/cmd/climc/shell" + modules "yunion.io/x/onecloud/pkg/mcclient/modules/llm" + options "yunion.io/x/onecloud/pkg/mcclient/options/llm" +) + +func init() { + cmd := shell.NewResourceCmd(&modules.Difies) + cmd.BatchCreate(new(options.DifyCreateOptions)) + cmd.List(new(options.DifyListOptions)) + cmd.Show(new(options.DifyShowOptions)) + cmd.Delete(new(options.DifyDeleteOptions)) + cmd.BatchPerform("stop", new(options.DifyStopOptions)) + cmd.BatchPerform("start", new(options.DifyStartOptions)) +} diff --git a/cmd/climc/shell/llm/dify_model.go b/cmd/climc/shell/llm/dify_model.go new file mode 100644 index 0000000000..b37db81a9d --- /dev/null +++ b/cmd/climc/shell/llm/dify_model.go @@ -0,0 +1,20 @@ +package llm + +import ( + "yunion.io/x/onecloud/cmd/climc/shell" + modules "yunion.io/x/onecloud/pkg/mcclient/modules/llm" + base_options "yunion.io/x/onecloud/pkg/mcclient/options" + options "yunion.io/x/onecloud/pkg/mcclient/options/llm" +) + +func init() { + cmd := shell.NewResourceCmd(&modules.DifyModel) + cmd.List(new(options.DifyModelListOptions)) + cmd.Show(new(options.DifyModelShowOptions)) + cmd.Update(new(options.DifyModelUpdateOptions)) + cmd.Create(new(options.DifyModelCreateOptions)) + cmd.Delete(new(options.DifyModelDeleteOptions)) + cmd.Perform("public", &base_options.BasePublicOptions{}) + cmd.Perform("private", &base_options.BaseIdOptions{}) + // cmd.Perform("clone", new(options.DesktopModelCloneOptions)) +} diff --git a/cmd/climc/shell/llm/image.go b/cmd/climc/shell/llm/image.go new file mode 100644 index 0000000000..d9a32712bf --- /dev/null +++ b/cmd/climc/shell/llm/image.go @@ -0,0 +1,19 @@ +package llm + +import ( + "yunion.io/x/onecloud/cmd/climc/shell" + modules "yunion.io/x/onecloud/pkg/mcclient/modules/llm" + base_options "yunion.io/x/onecloud/pkg/mcclient/options" + options "yunion.io/x/onecloud/pkg/mcclient/options/llm" +) + +func init() { + cmd := shell.NewResourceCmd(&modules.LLMImage) + cmd.List(new(options.LLMImageListOptions)) + cmd.Show(new(options.LLMImageShowOptions)) + cmd.Create(new(options.LLMImageCreateOptions)) + cmd.Update(new(options.LLMImageUpdateOptions)) + cmd.Delete(new(options.LLMImageDeleteOptions)) + cmd.Perform("public", &base_options.BasePublicOptions{}) + cmd.Perform("private", &base_options.BaseIdOptions{}) +} diff --git a/cmd/climc/shell/llm/llm.go b/cmd/climc/shell/llm/llm.go new file mode 100644 index 0000000000..6353bbcf11 --- /dev/null +++ b/cmd/climc/shell/llm/llm.go @@ -0,0 +1,18 @@ +package llm + +import ( + "yunion.io/x/onecloud/cmd/climc/shell" + modules "yunion.io/x/onecloud/pkg/mcclient/modules/llm" + options "yunion.io/x/onecloud/pkg/mcclient/options/llm" +) + +func init() { + cmd := shell.NewResourceCmd(&modules.LLMs) + cmd.BatchCreate(new(options.LLMCreateOptions)) + cmd.List(new(options.LLMListOptions)) + cmd.Show(new(options.LLMShowOptions)) + cmd.Delete(new(options.LLMDeleteOptions)) + // cmd.Perform("change-model", new(options.LLMChangeModelOptions)) + cmd.BatchPerform("stop", new(options.LLMStopOptions)) + cmd.BatchPerform("start", new(options.LLMStartOptions)) +} diff --git a/cmd/climc/shell/llm/llm_model.go b/cmd/climc/shell/llm/llm_model.go new file mode 100644 index 0000000000..6472c74c0e --- /dev/null +++ b/cmd/climc/shell/llm/llm_model.go @@ -0,0 +1,20 @@ +package llm + +import ( + "yunion.io/x/onecloud/cmd/climc/shell" + modules "yunion.io/x/onecloud/pkg/mcclient/modules/llm" + base_options "yunion.io/x/onecloud/pkg/mcclient/options" + options "yunion.io/x/onecloud/pkg/mcclient/options/llm" +) + +func init() { + cmd := shell.NewResourceCmd(&modules.LLMModel) + cmd.List(new(options.LLMModelListOptions)) + cmd.Show(new(options.LLMModelShowOptions)) + cmd.Update(new(options.LLMModelUpdateOptions)) + cmd.Create(new(options.LLMModelCreateOptions)) + cmd.Delete(new(options.LLMModelDeleteOptions)) + cmd.Perform("public", &base_options.BasePublicOptions{}) + cmd.Perform("private", &base_options.BaseIdOptions{}) + // cmd.Perform("clone", new(options.DesktopModelCloneOptions)) +} diff --git a/cmd/llm/main.go b/cmd/llm/main.go new file mode 100644 index 0000000000..3d040ebba0 --- /dev/null +++ b/cmd/llm/main.go @@ -0,0 +1,12 @@ +package main + +import ( + "yunion.io/x/onecloud/pkg/llm/service" + "yunion.io/x/onecloud/pkg/util/atexit" +) + +func main() { + defer atexit.Handle() + + service.StartService() +} diff --git a/pkg/apis/const.go b/pkg/apis/const.go index ef80357a48..351ac01807 100644 --- a/pkg/apis/const.go +++ b/pkg/apis/const.go @@ -45,6 +45,8 @@ const ( SERVICE_TYPE_APIMAP = "apimap" + SERVICE_TYPE_LLM = "llm" + STATUS_UPDATE_TAGS = "update_tags" STATUS_UPDATE_TAGS_FAILED = "update_tags_fail" diff --git a/pkg/apis/llm/dify.go b/pkg/apis/llm/dify.go new file mode 100644 index 0000000000..b057581057 --- /dev/null +++ b/pkg/apis/llm/dify.go @@ -0,0 +1,24 @@ +package llm + +type DifyCustomizedEnv struct { + Key string `json:"key"` + Value string `json:"value"` +} + +type DifyCustomized struct { + // Define custom environment variables here + CustomizedEnvs []*DifyCustomizedEnv `json:"customized_envs,omitempty"` + Registry string `json:"registry"` +} + +type DifyListInput struct { + LLMBaseListInput + + DifyModel string `json:"dify_model"` +} + +type DifyCreateInput struct { + LLMBaseCreateInput + + DifyModelId string +} diff --git a/pkg/apis/llm/dify_consts.go b/pkg/apis/llm/dify_consts.go new file mode 100644 index 0000000000..f3fdd3cccb --- /dev/null +++ b/pkg/apis/llm/dify_consts.go @@ -0,0 +1,195 @@ +package llm + +import ( + "crypto/rand" + "encoding/base64" + "encoding/hex" +) + +var ( + DIFY_SECRET_KEY string + DIFY_API_INNER_KEY string + DIFY_PLUGIN_SERVER_KEY string + DIFY_WEAVIATE_AUTHENTICATION_APIKEY_ALLOWED_KEYS string +) + +func init() { + skBytes := make([]byte, 32) + rand.Read(skBytes) + DIFY_SECRET_KEY = "sk-" + hex.EncodeToString(skBytes) + + innerKeyBytes := make([]byte, 32) + rand.Read(innerKeyBytes) + DIFY_API_INNER_KEY = base64.StdEncoding.EncodeToString(innerKeyBytes) + + pluginKeyBytes := make([]byte, 32) + rand.Read(pluginKeyBytes) + DIFY_PLUGIN_SERVER_KEY = base64.StdEncoding.EncodeToString(pluginKeyBytes) + + weaviateKeyBytes := make([]byte, 32) + rand.Read(weaviateKeyBytes) + DIFY_WEAVIATE_AUTHENTICATION_APIKEY_ALLOWED_KEYS = base64.URLEncoding.EncodeToString(weaviateKeyBytes) +} + +const ( + DIFY_POSTGRES_KEY = "postgres" + DIFY_REDIS_KEY = "redis" + DIFY_API_KEY = "api" + DIFY_WORKER_KEY = "worker" + DIFY_WORKER_BEAT_KEY = "beat" + DIFY_PLUGIN_KEY = "plugin" + DIFY_WEB_KEY = "web" + DIFY_SSRF_KEY = "ssrf" + DIFY_NGINX_KEY = "nginx" + DIFY_WEAVIATE_KEY = "weaviate" + DIFY_SANDBOX_KEY = "sandbox" +) + +const ( + DIFY_POSTGRES_IMAGE = "/postgres:15-alpine" + DIFY_REDIS_IMAGE = "/redis:6-alpine" + DIFY_NGINX_IMAGE = "/nginx:latest" + DIFY_API_IMAGE = "/langgenius/dify-api:1.7.2" + DIFY_PLUGIN_IMAGE = "/langgenius/dify-plugin-daemon:0.2.0-local" + DIFY_WEB_IMAGE = "/langgenius/dify-web:1.7.2" + DIFY_SANDBOX_IMAGE = "/langgenius/dify-sandbox:0.2.12" + DIFY_SSRF_IMAGE = "/ubuntu/squid:latest" + DIFY_WEAVIATE_IAMGE = "/semitechnologies/weaviate:1.19.0" +) + +const ( + DIFY_LOCALHOST = "localhost" + PIP_MIRROR_URL = "https://mirrors.aliyun.com/pypi/simple" +) + +const ( + DIFY_POSTGRES_PVC_MOUNT_PATH = "/var/lib/postgresql/data" + DIFY_POSTGRES_MAX_CONNECTIONS = "100" + DIFY_POSTGRES_SHARED_BUFFERS = "128MB" + DIFY_POSTGRES_WORK_MEM = "4MB" + DIFY_POSTGRES_MAINTENANCE_WORK_MEM = "64MB" + DIFY_POSTGRES_EFFECTIVE_CACHE_SIZE = "4096MB" + DIFY_POSTGRES_USER = "postgres" + DIFY_POSTGRES_PASSWORD = "difyai123456" + DIFY_POSTGRES_DB = "dify" + DIFY_POSTGRES_PGDATA = "/pgdata" + DIFY_POSTGRES_PORT = "5432" +) + +const ( + DIFY_REDISCLI_AUTH = "difyai123456" + DIFY_REDIS_PVC_MOUNT_PATH = "/data" + DIFY_REDIS_PORT = "6379" +) + +const ( + DIFY_API_PVC_MOUNT_PATH = "/app/api/storage" + DIFY_API_MODE = "api" + DIFY_API_SENTRY_DSN = "" + DIFY_API_SENTRY_TRACES_SAMPLE_RATE = "1.0" + DIFY_API_SENTRY_PROFILES_SAMPLE_RATE = "1.0" +) + +const ( + DIFY_WORKER_MODE = "worker" + DIFY_WORKER_BEAT_MODE = "beat" +) + +const ( + DIFY_PLUGIN_DB_DATABASE = "dify_plugin" + DIFY_PLUGIN_SERVER_PORT = "5002" + DIFY_PLUGIN_MAX_PACKAGE_SIZE = "52428800" + DIFY_PLUGIN_PPROF_ENABLED = "false" + DIFY_PLUGIN_DIFY_INNER_API_URL = "http://localhost:5001" + DIFY_PLUGIN_REMOTE_INSTALLING_HOST = "0.0.0.0" + DIFY_PLUGIN_REMOTE_INSTALLING_PORT = "5003" + DIFY_PLUGIN_WORKING_PATH = "/app/storage/cwd" + DIFY_PLUGIN_FORCE_VERIFYING_SIGNATURE = "true" + DIFY_PLUGIN_PYTHON_ENV_INIT_TIMEOUT = "120" + DIFY_PLUGIN_MAX_EXECUTION_TIMEOUT = "600" + DIFY_PLUGIN_STORAGE_TYPE = "local" + DIFY_PLUGIN_STORAGE_LOCAL_ROOT = "/app/storage" + DIFY_PLUGIN_INSTALLED_PATH = "plugin" + DIFY_PLUGIN_PACKAGE_CACHE_PATH = "plugin_packages" + DIFY_PLUGIN_MEDIA_CACHE_PATH = "assets" +) + +const ( + DIFY_WEB_CONSOLE_API_URL = "" + DIFY_WEB_APP_API_URL = "" + DIFY_WEB_SENTRY_DSN = "" + DIFY_WEB_NEXT_TELEMETRY_DISABLED = "0" + DIFY_WEB_TEXT_GENERATION_TIMEOUT_MS = "60000" + DIFY_WEB_CSP_WHITELIST = "" + DIFY_WEB_ALLOW_EMBED = "false" + DIFY_WEB_ALLOW_UNSAFE_DATA_SCHEME = "false" + DIFY_WEB_MARKETPLACE_API_URL = "https://marketplace.dify.ai" + DIFY_WEB_MARKETPLACE_URL = "https://marketplace.dify.ai" + DIFY_WEB_TOP_K_MAX_VALUE = "10" + DIFY_WEB_INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH = "4000" + DIFY_WEB_PM2_INSTANCES = "2" + DIFY_WEB_LOOP_NODE_MAX_COUNT = "100" + DIFY_WEB_MAX_TOOLS_NUM = "10" + DIFY_WEB_MAX_PARALLEL_LIMIT = "10" + DIFY_WEB_MAX_ITERATIONS_NUM = "99" + DIFY_WEB_ENABLE_WEBSITE_JINAREADER = "true" + DIFY_WEB_ENABLE_WEBSITE_FIRECRAWL = "true" + DIFY_WEB_ENABLE_WEBSITE_WATERCRAWL = "true" +) + +const ( + DIFY_SSRF_MOUNT_PATH = "/etc/squid/" + DIFY_SSRF_HTTP_PORT = "3128" + DIFY_SSRF_COREDUMP_DIR = "/var/spool/squid" +) + +const ( + DIFY_NGINX_MOUNT_PATH = "/etc/nginx/conf.d" + DIFY_NGINX_SERVER_NAME = "_" + DIFY_NGINX_PORT = "80" + DIFY_NGINX_WORKER_PROCESSES = "auto" + DIFY_NGINX_CLIENT_MAX_BODY_SIZE = "100M" + DIFY_NGINX_KEEPALIVE_TIMEOUT = "65" + DIFY_NGINX_PROXY_READ_TIMEOUT = "3600s" + DIFY_NGINX_PROXY_SEND_TIMEOUT = "3600s" +) + +const ( + DIFY_WEAVIATE_PERSISTENCE_DATA_PATH = "/var/lib/weaviate" + DIFY_WEAVIATE_QUERY_DEFAULTS_LIMIT = "25" + DIFY_WEAVIATE_AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED = "true" + DIFY_WEAVIATE_DEFAULT_VECTORIZER_MODULE = "none" + DIFY_WEAVIATE_CLUSTER_HOSTNAME = "node1" + DIFY_WEAVIATE_AUTHENTICATION_APIKEY_ENABLED = "true" + DIFY_WEAVIATE_AUTHENTICATION_APIKEY_USERS = "hello@dify.ai" + DIFY_WEAVIATE_AUTHORIZATION_ADMINLIST_ENABLED = "true" + DIFY_WEAVIATE_AUTHORIZATION_ADMINLIST_USERS = "hello@dify.ai" +) + +const ( + DIFY_SANDBOX_CONF_MOUNT_PATH = "/conf" + DIFY_SANDBOX_DEP_MOUNT_PATH = "/dependencies" + DIFY_SANDBOX_API_KEY = "dify-sandbox" + DIFY_SANDBOX_GIN_MODE = "release" + DIFY_SANDBOX_WORKER_TIMEOUT = "15" + DIFY_SANDBOX_ENABLE_NETWORK = "true" + DIFY_SANDBOX_HTTP_PROXY = "http://" + DIFY_LOCALHOST + ":" + DIFY_SSRF_HTTP_PORT + DIFY_SANDBOX_HTTPS_PROXY = "http://" + DIFY_LOCALHOST + ":" + DIFY_SSRF_HTTP_PORT + DIFY_SANDBOX_PORT = "8194" +) + +const ( + DIFY_DEPLOY_REDIS_FAILED = "dify_deploy_redis_failed" + DIFY_DEPLOY_POSTGRES_FAILED = "dify_deploy_postgres_failed" + DIFY_DEPLOY_API_FAILED = "dify_deploy_api_failed" + DIFY_DEPLOY_WORKER_FAILED = "dify_deploy_worker_failed" + DIFY_DEPLOY_WORKER_BEAT_FAILED = "dify_deploy_worker_beat_failed" + DIFY_DEPLOY_WEB_FAILED = "dify_deploy_web_failed" + DIFY_DEPLOY_PLUGIN_FAILED = "dify_deploy_plugin_failed" + DIFY_DEPLOY_SANDBOX_FAILED = "dify_deploy_sandbox_failed" + DIFY_DEPLOY_SSRF_FAILED = "dify_deploy_ssrf_failed" + DIFY_DEPLOY_NGINX_FAILED = "dify_deploy_nginx_failed" + DIFY_DEPLOY_WEAVIATE_FAILED = "dify_deploy_weaviate_failed" + DIFY_CREATE_FAILED = "create_dify_failed" + DIFY_CREATED = "running" +) diff --git a/pkg/apis/llm/dify_files.go b/pkg/apis/llm/dify_files.go new file mode 100644 index 0000000000..dfdf2ab653 --- /dev/null +++ b/pkg/apis/llm/dify_files.go @@ -0,0 +1,290 @@ +package llm + +const ( + DIFY_SSRF_ENTRYPINT_SHELL = ` +mkdir -p /etc/squid + +cat > /etc/squid/squid.conf.template <<'EOF' +%s +EOF + +echo "[ENTRYPOINT] re-create snakeoil self-signed certificate removed in the build process" +if [ ! -f /etc/ssl/private/ssl-cert-snakeoil.key ]; then + /usr/sbin/make-ssl-cert generate-default-snakeoil --force-overwrite > /dev/null 2>&1 +fi + +tail -F /var/log/squid/access.log 2>/dev/null & +tail -F /var/log/squid/error.log 2>/dev/null & +tail -F /var/log/squid/store.log 2>/dev/null & +tail -F /var/log/squid/cache.log 2>/dev/null & + +echo "[ENTRYPOINT] replacing environment variables in the template" +awk '{ + while(match($0, /\${[A-Za-z_][A-Za-z_0-9]*}/)) { + var = substr($0, RSTART+2, RLENGTH-3) + val = ENVIRON[var] + $0 = substr($0, 1, RSTART-1) val substr($0, RSTART+RLENGTH) + } + print +}' /etc/squid/squid.conf.template > /etc/squid/squid.conf + +chown -R "$SQUID_USER":"$SQUID_USER" /etc/squid + +/usr/sbin/squid -Nz +echo "[ENTRYPOINT] starting squid" +/usr/sbin/squid -f /etc/squid/squid.conf -NYC 1 +` + DIFY_SSRF_SQUID_CONFIGURATION_FILE = `visible_hostname localhost # set visible_hostname to avoid WARNING: Could not determine this machines public hostname. +acl localnet src 0.0.0.1-0.255.255.255 # RFC 1122 "this" network (LAN) +acl localnet src 10.0.0.0/8 # RFC 1918 local private network (LAN) +acl localnet src 100.64.0.0/10 # RFC 6598 shared address space (CGN) +acl localnet src 169.254.0.0/16 # RFC 3927 link-local (directly plugged) machines +acl localnet src 172.16.0.0/12 # RFC 1918 local private network (LAN) +acl localnet src 192.168.0.0/16 # RFC 1918 local private network (LAN) +acl localnet src fc00::/7 # RFC 4193 local private network range +acl localnet src fe80::/10 # RFC 4291 link-local (directly plugged) machines +acl SSL_ports port 443 +# acl SSL_ports port 1025-65535 # Enable the configuration to resolve this issue: https://github.com/langgenius/dify/issues/12792 +acl Safe_ports port 80 # http +acl Safe_ports port 21 # ftp +acl Safe_ports port 443 # https +acl Safe_ports port 70 # gopher +acl Safe_ports port 210 # wais +acl Safe_ports port 1025-65535 # unregistered ports +acl Safe_ports port 280 # http-mgmt +acl Safe_ports port 488 # gss-http +acl Safe_ports port 591 # filemaker +acl Safe_ports port 777 # multiling http +acl CONNECT method CONNECT +acl allowed_domains dstdomain .marketplace.dify.ai +http_access allow allowed_domains +http_access deny !Safe_ports +http_access deny CONNECT !SSL_ports +http_access allow localhost manager +http_access deny manager +http_access allow localhost +include /etc/squid/conf.d/*.conf +http_access deny all + +################################## Proxy Server ################################ +http_port ${HTTP_PORT} +coredump_dir ${COREDUMP_DIR} +refresh_pattern ^ftp: 1440 20% 10080 +refresh_pattern ^gopher: 1440 0% 1440 +refresh_pattern -i (/cgi-bin/|\?) 0 0% 0 +refresh_pattern \/(Packages|Sources)(|\.bz2|\.gz|\.xz)$ 0 0% 0 refresh-ims +refresh_pattern \/Release(|\.gpg)$ 0 0% 0 refresh-ims +refresh_pattern \/InRelease$ 0 0% 0 refresh-ims +refresh_pattern \/(Translation-.*)(|\.bz2|\.gz|\.xz)$ 0 0% 0 refresh-ims +refresh_pattern . 0 20% 4320 + + +# cache_dir ufs /var/spool/squid 100 16 256 +# upstream proxy, set to your own upstream proxy IP to avoid SSRF attacks +# cache_peer 172.1.1.1 parent 3128 0 no-query no-digest no-netdb-exchange default + +################################## Reverse Proxy To Sandbox ################################ +http_port ${REVERSE_PROXY_PORT} accel vhost +cache_peer ${SANDBOX_HOST} parent ${SANDBOX_PORT} 0 no-query originserver +acl src_all src all +http_access allow src_all + +# Unless the option's size is increased, an error will occur when uploading more than two files. +client_request_buffer_max_size 100 MB +` +) + +const ( + DIFY_NGINX_ENTRYPINT_SHELL = ` +mkdir -p /etc/nginx +cat > /etc/nginx/nginx.conf.template <<'EOF' +%s +EOF + +cat > /etc/nginx/proxy.conf.template <<'EOF' +%s +EOF + +cat > /etc/nginx/conf.d/default.conf.template <<'EOF' +%s +EOF + +env_vars=$(printenv | cut -d= -f1 | sed 's/^/$/g' | paste -sd, -) +echo "Substituting variables: $env_vars" + +envsubst "$env_vars" < /etc/nginx/nginx.conf.template > /etc/nginx/nginx.conf +envsubst "$env_vars" < /etc/nginx/proxy.conf.template > /etc/nginx/proxy.conf +envsubst "$env_vars" < /etc/nginx/conf.d/default.conf.template > /etc/nginx/conf.d/default.conf + +exec nginx -g 'daemon off;'` + DIFY_NGINX_NGINX_CONF_FILE = ` +user nginx; +worker_processes ${NGINX_WORKER_PROCESSES}; + +error_log /var/log/nginx/error.log notice; +pid /var/run/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout ${NGINX_KEEPALIVE_TIMEOUT}; + + #gzip on; + client_max_body_size ${NGINX_CLIENT_MAX_BODY_SIZE}; + + include /etc/nginx/conf.d/*.conf; +} +` + DIFY_NGINX_PROXY_CONF_FILE = ` +proxy_set_header Host $host; +proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; +proxy_set_header X-Forwarded-Proto $scheme; +proxy_set_header X-Forwarded-Port $server_port; +proxy_http_version 1.1; +proxy_set_header Connection ""; +proxy_buffering off; +proxy_read_timeout ${NGINX_PROXY_READ_TIMEOUT}; +proxy_send_timeout ${NGINX_PROXY_SEND_TIMEOUT}; +` + DIFY_NGINX_DEFAULT_CONF_FILE = `# Please do not directly edit this file. Instead, modify the .env variables related to NGINX configuration. + +server { + listen ${NGINX_PORT}; + server_name ${NGINX_SERVER_NAME}; + + location /console/api { + proxy_pass http://localhost:5001; + include proxy.conf; + } + + location /api { + proxy_pass http://localhost:5001; + include proxy.conf; + } + + location /v1 { + proxy_pass http://localhost:5001; + include proxy.conf; + } + + location /files { + proxy_pass http://localhost:5001; + include proxy.conf; + } + + location /explore { + proxy_pass http://localhost:3000; + include proxy.conf; + } + + location /e/ { + proxy_pass http://localhost:5002; + proxy_set_header Dify-Hook-Url $scheme://$host$request_uri; + include proxy.conf; + } + + location / { + proxy_pass http://localhost:3000; + include proxy.conf; + } + location /mcp { + proxy_pass http://localhost:5001; + include proxy.conf; + } + # placeholder for acme challenge location + # ${ACME_CHALLENGE_LOCATION} + + # placeholder for https config defined in https.conf.template + # ${HTTPS_CONFIG} +} +` +) + +const ( + DIFY_SANDBOX_WRITE_CONF_SHELL = ` +cat > /conf/config.yaml <<'EOF' +%s +EOF + +cat > /conf/config.yaml.template <<'EOF' +%s +EOF + +touch /dependencies/python-requirements.txt + +# decompress nodejs +tar -xvf $NODE_TAR_XZ -C /opt +ln -s $NODE_DIR/bin/node /usr/local/bin/node +rm -f $NODE_TAR_XZ + +# start main +/main + ` + DIFY_SANDBOX_CONF_FILE = `app: + port: 8194 + debug: True + key: dify-sandbox +max_workers: 4 +max_requests: 50 +worker_timeout: 5 +python_path: /usr/local/bin/python3 +enable_network: True # please make sure there is no network risk in your environment +allowed_syscalls: # please leave it empty if you have no idea how seccomp works +proxy: + socks5: '' + http: '' + https: '' +` + DIFY_SANDBOX_CONF_TEMP_FILE = `app: + port: 8194 + debug: True + key: dify-sandbox +max_workers: 4 +max_requests: 50 +worker_timeout: 5 +python_path: /usr/local/bin/python3 +python_lib_path: + - /usr/local/lib/python3.10 + - /usr/lib/python3.10 + - /usr/lib/python3 + - /usr/lib/x86_64-linux-gnu + - /etc/ssl/certs/ca-certificates.crt + - /etc/nsswitch.conf + - /etc/hosts + - /etc/resolv.conf + - /run/systemd/resolve/stub-resolv.conf + - /run/resolvconf/resolv.conf + - /etc/localtime + - /usr/share/zoneinfo + - /etc/timezone + # add more paths if needed +python_pip_mirror_url: https://pypi.tuna.tsinghua.edu.cn/simple +nodejs_path: /usr/local/bin/node +enable_network: True +allowed_syscalls: + - 1 + - 2 + - 3 + # add all the syscalls which you require +proxy: + socks5: '' + http: '' + https: '' +` +) diff --git a/pkg/apis/llm/doc.go b/pkg/apis/llm/doc.go new file mode 100644 index 0000000000..d613a855bd --- /dev/null +++ b/pkg/apis/llm/doc.go @@ -0,0 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package llm // import "yunion.io/x/onecloud/pkg/apis/llm" diff --git a/pkg/apis/llm/image.go b/pkg/apis/llm/image.go new file mode 100644 index 0000000000..f84c801924 --- /dev/null +++ b/pkg/apis/llm/image.go @@ -0,0 +1,26 @@ +package llm + +import "yunion.io/x/onecloud/pkg/apis" + +type LLMImageListInput struct { + apis.SharableVirtualResourceListInput + + ImageLabel string `json:"image_label"` + ImageName string `json:"image_name"` +} + +type LLMImageCreateInput struct { + apis.SharableVirtualResourceCreateInput + + ImageName string `json:"image_name"` + ImageLabel string `json:"image_label"` + CredentialId string `json:"credential_id"` +} + +type LLMImageUpdateInput struct { + apis.SharableVirtualResourceCreateInput + + ImageName *string `json:"image_name,omitempty"` + ImageLabel *string `json:"image_label,omitempty"` + CredentialId *string `json:"credential_id,omitempty"` +} diff --git a/pkg/apis/llm/llm.go b/pkg/apis/llm/llm.go new file mode 100644 index 0000000000..90e41921cc --- /dev/null +++ b/pkg/apis/llm/llm.go @@ -0,0 +1,45 @@ +package llm + +import "yunion.io/x/onecloud/pkg/apis" + +const ( + SERVICE_TYPE = "llm" +) + +type LLMBaseCreateInput struct { + apis.VirtualResourceCreateInput + + PreferHost string + AutoStart bool + BandwidthMB int `json:"bandwidth_mb"` + DebugMode bool `json:"debug_mode"` + RootfsUnlimit bool `json:"rootfs_unlimit"` +} + +type LLMCreateInput struct { + LLMBaseCreateInput + + LLMModelId string + LLMImageId string +} + +type LLMBaseListInput struct { + apis.VirtualResourceListInput + apis.EnabledResourceBaseListInput + + Host string `json:"host"` + Status []string `json:"status"` + + NoVolume *bool `json:"no_volume"` + ListenPort int `json:"listen_port"` + PublicIp string `json:"public_ip"` + VolumeId string `json:"volume_id"` + Unused *bool `json:"unused"` +} + +type LLMListInput struct { + LLMBaseListInput + + LLMModel string `json:"llm_model"` + LLMImage string `json:"llm_image"` +} diff --git a/pkg/apis/llm/llm_const.go b/pkg/apis/llm/llm_const.go new file mode 100644 index 0000000000..87ec78f4d4 --- /dev/null +++ b/pkg/apis/llm/llm_const.go @@ -0,0 +1,39 @@ +package llm + +const ( + STATUS_READY = "ready" +) + +const ( + /* 未知 */ + LLM_STATUS_UNKOWN = "unkown" + + /* 创建失败 */ + LLM_STATUS_CREATE_FAIL = "create_fail" + + /* 启动失败 */ + LLM_STATUS_START_FAIL = "start_fail" + /* 停机失败 */ + LLM_STATUS_STOP_FAILED = "stop_fail" + + /* 停机 */ + LLM_STATUS_READY = "ready" + /* 运行 */ + LLM_STATUS_RUNNING = "running" + + LLM_STATUS_CREATING_POD = "creating_pod" + LLM_STATUS_CREAT_POD_FAILED = "creat_pod_failed" + LLM_STATUS_PULLING_MODEL = "pulling_model" + LLM_STATUS_GET_MANIFESTS_FAILED = "get_manifests_failed" + LLM_STATUS_DOWNLOADING_BLOBS = "downloading_blobs" + LLM_STATUS_DOWNLOADING_BLOBS_FAILED = "downloading_blobs_failed" + LLM_STATUS_FETCHING_GGUF_FILE = "fetching_gguf_file" + LLM_STATUS_FETCH_GGUF_FILE_FAILED = "fetch_gguf_failed" + LLM_STATUS_CREATING_GGUF_MODEL = "creating_gguf_model" + LLM_STATUS_CREATE_GGUF_MODEL_FAILED = "create_gguf_model_failed" + LLM_STATUS_PULLED_MODEL = "pulled_model" + LLM_STATUS_PULL_MODEL_FAILED = "pull_model_failed" + LLM_STATUS_START_DELETE = "start_delete" + LLM_STATUS_DELETING = "deleting" + LLM_STATUS_DELETE_FAILED = "delete_fail" +) diff --git a/pkg/apis/llm/llm_container.go b/pkg/apis/llm/llm_container.go new file mode 100644 index 0000000000..9440abd308 --- /dev/null +++ b/pkg/apis/llm/llm_container.go @@ -0,0 +1,30 @@ +package llm + +import ( + "yunion.io/x/pkg/util/sets" + + "yunion.io/x/onecloud/pkg/apis" +) + +type LLMContainerType string + +const ( + LLM_CONTAINER_OLLAMA LLMContainerType = "ollama" +) + +var ( + LLM_CONTAINER_TYPES = sets.NewString( + string(LLM_CONTAINER_OLLAMA), + ) +) + +func IsLLMContainerType(t string) bool { + return LLM_CONTAINER_TYPES.Has(t) +} + +type LLMContainerCreateInput struct { + apis.VirtualResourceCreateInput + LLMId string `json:"llm_id"` + Type string `json:"type"` + CmpId string `json:"cmp_id"` +} diff --git a/pkg/apis/llm/model.go b/pkg/apis/llm/model.go new file mode 100644 index 0000000000..cfe9568d53 --- /dev/null +++ b/pkg/apis/llm/model.go @@ -0,0 +1,211 @@ +package llm + +import ( + "reflect" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/gotypes" + + "yunion.io/x/onecloud/pkg/apis" + computeapi "yunion.io/x/onecloud/pkg/apis/compute" +) + +type HostInfo struct { + HostId string + Host string + HostAccessIp string + HostEIP string +} + +func init() { + gotypes.RegisterSerializable(reflect.TypeOf(&PortMappings{}), func() gotypes.ISerializable { + return &PortMappings{} + }) + gotypes.RegisterSerializable(reflect.TypeOf(&Devices{}), func() gotypes.ISerializable { + return &Devices{} + }) + gotypes.RegisterSerializable(reflect.TypeOf(&PortMappingEnvs{}), func() gotypes.ISerializable { + return &PortMappingEnvs{} + }) + gotypes.RegisterSerializable(reflect.TypeOf(&Envs{}), func() gotypes.ISerializable { + return &Envs{} + }) +} + +type PortMappingEnv struct { + Key string `json:"key"` + ValueFrom string `json:"value_from"` +} + +type PortMappingEnvs []PortMappingEnv + +func (pm PortMappingEnvs) String() string { + return jsonutils.Marshal(pm).String() +} + +func (pm PortMappingEnvs) IsZero() bool { + return len(pm) == 0 +} + +type PortMapping struct { + Protocol string `json:"protocol"` + ContainerPort int `json:"container_port"` + RemoteIps []string `json:"remote_ips"` + FirstPortOffset *int `json:"first_port_offset"` + Envs []computeapi.GuestPortMappingEnv `json:"envs"` +} + +type PortMappings []PortMapping + +func (s PortMappings) String() string { + return jsonutils.Marshal(s).String() +} + +func (s PortMappings) IsZero() bool { + return len(s) == 0 +} + +type Device struct { + DevType string `json:"dev_type"` + Model string `json:"model"` + DevicePath string `json:"device_path"` +} + +type Devices []Device + +func (s Devices) String() string { + return jsonutils.Marshal(s).String() +} + +func (s Devices) IsZero() bool { + return len(s) == 0 +} + +type Env struct { + Key string `json:"key"` + Value string `json:"value"` +} + +type Envs []Env + +func (s Envs) String() string { + return jsonutils.Marshal(s).String() +} + +func (s Envs) IsZero() bool { + return len(s) == 0 +} + +type LLMModelDetails struct { + apis.SharableVirtualResourceDetails + // 当前大模型套餐包含的实例个数。 + LLMCapacity int + Image string + ImageLabel string + ImageName string + + Template string +} + +// type MountedAppResourceDetails struct { +// MountedApps []string `json:"mounted_apps"` +// } + +type LLMModelBaseCreateInput struct { + apis.SharableVirtualResourceCreateInput + + Cpu int `json:"cpu"` + Memory int `json:"memory"` + + Bandwidth int `json:"bandwidth"` + + Volumes *Volumes `json:"volumes"` + PortMappings *PortMappings `json:"port_mappings"` + Devices *Devices `json:"devices"` + Envs *Envs `json:"envs"` + Properties map[string]string `json:"properties"` +} + +type LLMModelBaseUpdateInput struct { + apis.SharableVirtualResourceBaseUpdateInput + // MountedAppResourceUpdateInput + + Cpu *int `json:"cpu"` + Memory *int `json:"memory"` + + // RequstSyncImage *bool `json:"request_sync_image"` + + DiskSizeMB *int `json:"disk_size_mb"` + TemplateId *string `json:"template_id"` + StorageType *string `json:"storage_type"` + Volumes *Volumes `json:"volumes"` + + Bandwidth *int `json:"bandwidth"` + PortMappings *PortMappings `json:"port_mappings"` + Devices *Devices `json:"devices"` + Envs *Envs `json:"envs"` + Properties map[string]string `json:"properties"` +} + +type LLMModelListInput struct { + apis.SharableVirtualResourceListInput + // MountedAppResourceListInput + + LLMType string `json:"llm_type"` +} + +type LLMModelCreateInput struct { + LLMModelBaseCreateInput + + LLMImageId string `json:"llm_image_id"` + LLMType string `json:"llm_type"` + LLMModelName string `json:"llm_model_name"` +} + +type LLMModelUpdateInput struct { + LLMModelBaseUpdateInput + + LLMImageId string `json:"llm_image_id"` + LLMModelName string `json:"llm_model_name"` +} + +// type LLMModelCloneInput struct { +// Name string `json:"name"` +// } + +// type LLMModelSyncImageRequestTaskInput struct { +// Request bool `json:"request"` +// } + +type DifyModelListInput struct { + apis.SharableVirtualResourceListInput + // MountedAppResourceListInput +} + +type DifyModelCreateInput struct { + LLMModelBaseCreateInput + + PostgresImageId string `json:"postgres_image_id"` + RedisImageId string `json:"redis_image_id"` + NginxImageId string `json:"nginx_image_id"` + DifyApiImageId string `json:"dify_api_image_id"` + DifyPluginImageId string `json:"dify_plugin_image_id"` + DifyWebImageId string `json:"dify_web_image_id"` + DifySandboxImageId string `json:"dify_sandbox_image_id"` + DifySSRFImageId string `json:"dify_ssrf_image_id"` + DifyWeaviateImageId string `json:"dify_weaviate_image_id"` +} + +type DifyModelUpdateInput struct { + LLMModelBaseUpdateInput + + PostgresImageId string `json:"postgres_image_id"` + RedisImageId string `json:"redis_image_id"` + NginxImageId string `json:"nginx_image_id"` + DifyApiImageId string `json:"dify_api_image_id"` + DifyPluginImageId string `json:"dify_plugin_image_id"` + DifyWebImageId string `json:"dify_web_image_id"` + DifySandboxImageId string `json:"dify_sandbox_image_id"` + DifySSRFImageId string `json:"dify_ssrf_image_id"` + DifyWeaviateImageId string `json:"dify_weaviate_image_id"` +} diff --git a/pkg/apis/llm/ollama.go b/pkg/apis/llm/ollama.go new file mode 100644 index 0000000000..6b181b0b4f --- /dev/null +++ b/pkg/apis/llm/ollama.go @@ -0,0 +1,111 @@ +package llm + +import ( + "fmt" + + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/apis" +) + +type OllamaModelFileSpec struct { + Parameter *OllamaModelFileParameter `json:"parameter"` + Template *string `json:"template,omitempty"` + System *string `json:"system,omitempty"` + // Adapter *string `json:"adapter,omitempty"` + License *string `json:"license,omitempty"` + Message []*OllamaModelFileMessage `json:"message,omitempty"` +} + +type OllamaGgufSpec struct { + GgufFile string `json:"gguf_file"` + Source string `json:"source"` + ModelFile *OllamaModelFileSpec `json:"modelfile,omitempty"` +} + +// type OllamaPullModelInput struct { +// Model string `json:"model"` +// Gguf *OllamaGgufSpec `json:"gguf,omitempty"` +// } + +// type OllamaCreateInput struct { +// compute.ServerCreateInput +// OllamaPullModelInput +// } + +type OllamaListInput struct { + apis.VirtualResourceListInput + GuestId string `json:"guest_id"` +} + +type OllamaAccessCacheInput struct { + ModelName string `json:"model_name"` + Blobs []string `json:"blobs"` +} + +type OllamaAccessGgufFileInput struct { + HostPath string `json:"host_path"` + TargetDir string `json:"target_dir"` +} + +type OllamaModelFileMessage struct { + Role string + Content string +} + +func (m *OllamaModelFileMessage) ValidateRole() error { + switch m.Role { + case LLM_OLLAMA_GGUF_MESSAGE_ROLE_SYSTEM, + LLM_OLLAMA_GGUF_MESSAGE_ROLE_USER, + LLM_OLLAMA_GGUF_MESSAGE_ROLE_ASSISTANT: + return nil + default: + return errors.Errorf("invalid role: %s, must be one of: system, user, assistant", m.Role) + } +} + +type OllamaModelFileParameter struct { + NumCtx *int `json:"num_ctx,omitempty"` + RepeatLastN *int `json:"repeat_last_n,omitempty"` + RepeatPenalty *float64 `json:"repeat_penalty,omitempty"` + Temperature *float64 `json:"temperature,omitempty"` + Seed *int `json:"seed,omitempty"` + Stop *string `json:"stop,omitempty"` + NumPredict *int `json:"num_predict,omitempty"` + TopK *int `json:"top_k,omitempty"` + TopP *float64 `json:"top_p,omitempty"` + MinP *float64 `json:"min_p,omitempty"` +} + +func (p *OllamaModelFileParameter) GetParameters() map[string]string { + pairs := make(map[string]string) + + addInt := func(key string, val *int) { + if val != nil { + pairs[key] = fmt.Sprintf("%d", *val) + } + } + addFloat := func(key string, val *float64) { + if val != nil { + pairs[key] = fmt.Sprintf("%f", *val) + } + } + addString := func(key string, val *string) { + if val != nil { + pairs[key] = fmt.Sprintf("\"%s\"", *val) + } + } + + addInt(LLM_OLLAMA_MODELFILE_PARAMETER_NUM_CTX, p.NumCtx) + addInt(LLM_OLLAMA_MODELFILE_PARAMETER_REPEAT_LAST_N, p.RepeatLastN) + addFloat(LLM_OLLAMA_MODELFILE_PARAMETER_REPEAT_PENALTY, p.RepeatPenalty) + addFloat(LLM_OLLAMA_MODELFILE_PARAMETER_TEMPERATURE, p.Temperature) + addInt(LLM_OLLAMA_MODELFILE_PARAMETER_SEED, p.Seed) + addString(LLM_OLLAMA_MODELFILE_PARAMETER_STOP, p.Stop) + addInt(LLM_OLLAMA_MODELFILE_PARAMETER_NUM_PREDICT, p.NumPredict) + addInt(LLM_OLLAMA_MODELFILE_PARAMETER_TOP_K, p.TopK) + addFloat(LLM_OLLAMA_MODELFILE_PARAMETER_TOP_P, p.TopP) + addFloat(LLM_OLLAMA_MODELFILE_PARAMETER_MIN_P, p.MinP) + + return pairs +} diff --git a/pkg/apis/llm/ollama_const.go b/pkg/apis/llm/ollama_const.go new file mode 100644 index 0000000000..e72135631f --- /dev/null +++ b/pkg/apis/llm/ollama_const.go @@ -0,0 +1,52 @@ +package llm + +const ( + LLM_OLLAMA = "ollama" + LLM_OLLAMA_EXEC_PATH = "/bin/ollama" + LLM_OLLAMA_PULL_ACTION = "pull" + LLM_OLLAMA_LIST_ACTION = "list" + LLM_OLLAMA_CREATE_ACTION = "create" + LLM_OLLAMA_EXPORT_ENV_KEY = "OLLAMA_HOST" + LLM_OLLAMA_EXPORT_ENV_VALUE = "0.0.0.0:11434" +) + +const ( + LLM_OLLAMA_HOST_PATH = "/opt/ollama-models" + LLM_OLLAMA_HOST_MANIFESTS_DIR = "/manifests" + LLM_OLLAMA_CACHE_DIR = "/.llm_ollama_cache" + LLM_OLLAMA_CACHE_MOUNT_PATH = "/usr/local" + LLM_OLLAMA_LIBRARY_BASE_URL = `https://registry.ollama.ai/v2/library/%s` + LLM_OLLAMA_BASE_PATH = "/root/.ollama/models" + LLM_OLLAMA_BLOBS_DIR = "/blobs" + LLM_OLLAMA_MANIFESTS_BASE_PATH = "/manifests/registry.ollama.ai/library" +) + +const ( + LLM_OLLAMA_GGUF_DIR = "/gguf" + LLM_OLLAMA_GGUF_SOURCE_HOST = "host" + LLM_OLLAMA_GGUF_SOURCE_WEB = "web" + LLM_OLLAMA_MODELFILE_NAME = "modelfile" + LLM_OLLAMA_GGUF_FROM = "FROM %s\n" + LLM_OLLAMA_GGUF_PARAMETER = "PARAMETER %s %s\n" + LLM_OLLAMA_GGUF_TEMPLATE = "TEMPLATE \"\"\"%s\"\"\"\n" + LLM_OLLAMA_GGUF_SYSTEM = "SYSTEM %s\n" + LLM_OLLAMA_GGUF_ADAPTER = "ADAPTER %s\n" + LLM_OLLAMA_GGUF_LICENSE = "LICENSE \"\"\"%s\"\"\"\n" + LLM_OLLAMA_GGUF_MESSAGE = "MESSAGE %s %s\n" + LLM_OLLAMA_GGUF_MESSAGE_ROLE_SYSTEM = "system" + LLM_OLLAMA_GGUF_MESSAGE_ROLE_USER = "user" + LLM_OLLAMA_GGUF_MESSAGE_ROLE_ASSISTANT = "assistant" +) + +const ( + LLM_OLLAMA_MODELFILE_PARAMETER_NUM_CTX = "num_ctx" + LLM_OLLAMA_MODELFILE_PARAMETER_REPEAT_LAST_N = "repeat_last_n" + LLM_OLLAMA_MODELFILE_PARAMETER_REPEAT_PENALTY = "repeat_penalty" + LLM_OLLAMA_MODELFILE_PARAMETER_TEMPERATURE = "temperature" + LLM_OLLAMA_MODELFILE_PARAMETER_SEED = "seed" + LLM_OLLAMA_MODELFILE_PARAMETER_STOP = "stop" + LLM_OLLAMA_MODELFILE_PARAMETER_NUM_PREDICT = "num_predict" + LLM_OLLAMA_MODELFILE_PARAMETER_TOP_K = "top_k" + LLM_OLLAMA_MODELFILE_PARAMETER_TOP_P = "top_p" + LLM_OLLAMA_MODELFILE_PARAMETER_MIN_P = "min_p" +) diff --git a/pkg/apis/llm/volume.go b/pkg/apis/llm/volume.go new file mode 100644 index 0000000000..c137b84d99 --- /dev/null +++ b/pkg/apis/llm/volume.go @@ -0,0 +1,158 @@ +package llm + +import ( + "fmt" + "reflect" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/gotypes" + + "yunion.io/x/onecloud/pkg/apis" + computeapi "yunion.io/x/onecloud/pkg/apis/compute" +) + +func init() { + gotypes.RegisterSerializable(reflect.TypeOf(&Volume{}), func() gotypes.ISerializable { + return &Volume{} + }) + gotypes.RegisterSerializable(reflect.TypeOf(&Volumes{}), func() gotypes.ISerializable { + return &Volumes{} + }) + gotypes.RegisterSerializable(reflect.TypeOf(&ContainerVolumeRelations{}), func() gotypes.ISerializable { + return &ContainerVolumeRelations{} + }) +} + +const ( + VOLUME_STATUS_READY = computeapi.DISK_READY + + VOLUME_STATUS_START_SYNC_STATUS = "start_syncstatus" + VOLUME_STATUS_SYNC_STATUS = "syncstatus" + VOLUME_STATUS_SYNC_STATUS_FAILED = "syncstatus_fail" + + VOLUME_STATUS_START_RESET = "start_reset" + VOLUME_STATUS_RESETTING = "resetting" + VOLUME_STATUS_RESET_FAILED = "reset_fail" + + VOLUME_STATUS_START_RESIZE = "start_resize" + VOLUME_STATUS_RESIZING = "resizing" + VOLUME_STATUS_RESIZE_FAILED = "resize_fail" + + VOLUME_STATUS_ASSIGNING = "assigning" +) + +type VolumeDetails struct { + apis.VirtualResourceDetails + + Volume + + Template string + + DesktopId string + DesktopName string + DesktopStatus string + + Disk string + + StorageId string + Storage string + StorageStatus string + StorageHosts []computeapi.StorageHost + + Hosts []HostInfo + + HostInfo + + InstanceId string + InstanceName string +} + +type ContainerVolumeRelation struct { + MountPath string `json:"mount_path"` + SubDirectory string `json:"sub_directory"` + Overlay *apis.ContainerVolumeMountDiskOverlay `json:"overlay"` + FsUser *int64 `json:"fs_user"` + FsGroup *int64 `json:"fs_group"` +} + +// key is string format of integer +type ContainerVolumeRelations map[string]*ContainerVolumeRelation + +func (s ContainerVolumeRelations) String() string { + return jsonutils.Marshal(s).String() +} + +func (s ContainerVolumeRelations) IsZero() bool { + return len(s) == 0 +} + +type Volume struct { + // db.SStandaloneAnonResourceBase + Id string + Name string + StorageType string + TemplateId string + SizeMB int + // Container index to mount path relation + Containers ContainerVolumeRelations `json:"containers"` +} + +func (s Volume) String() string { + return jsonutils.Marshal(s).String() +} + +func (s Volume) IsZero() bool { + return s.SizeMB == 0 +} + +func (s Volume) GetVolumeByContainer(containerIndex int) *ContainerVolumeRelation { + if len(s.Containers) == 0 { + return nil + } + key := fmt.Sprintf("%d", containerIndex) + return s.Containers[key] +} + +type Volumes []Volume + +func (s Volumes) String() string { + return jsonutils.Marshal(s).String() +} + +func (s Volumes) IsZero() bool { + return len(s) == 0 +} + +type VolumeCreateInput struct { + apis.VirtualResourceCreateInput + ImageId string + Size int + StorageType string + + PreferHost string +} + +type VolumePerformResetInput struct { + SizeGb int `json:"size_gb"` +} + +type VolumePerformResizeInput struct { + Size string `json:"size"` +} + +type VolumeResizeTaskInput struct { + SizeMB int `json:"size_mb"` + DesktopStatus string `json:"desktop_status"` +} + +type VolumeListInput struct { + apis.VirtualResourceListInput + + Host string `json:"host"` + + Unused *bool `json:"unused"` + + Size string `json:"size"` + + DesktopId string `json:"desktop_id"` +} diff --git a/pkg/llm/drivers/llm_container/doc.go b/pkg/llm/drivers/llm_container/doc.go new file mode 100644 index 0000000000..acd0398605 --- /dev/null +++ b/pkg/llm/drivers/llm_container/doc.go @@ -0,0 +1 @@ +package llm_container // import "yunion.io/x/onecloud/pkg/llm/drivers/llm_container" diff --git a/pkg/llm/drivers/llm_container/ollama.go b/pkg/llm/drivers/llm_container/ollama.go new file mode 100644 index 0000000000..63d87f8c0c --- /dev/null +++ b/pkg/llm/drivers/llm_container/ollama.go @@ -0,0 +1,237 @@ +package llm_container + +import ( + "context" + "fmt" + "path" + "regexp" + "strings" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/apis" + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/llm/models" + "yunion.io/x/onecloud/pkg/llm/utils" + "yunion.io/x/onecloud/pkg/mcclient" +) + +func init() { + models.RegisterLLMContainerDriver(newOllama()) + // log.Infoln("registed ollama") +} + +type ollama struct{} + +func newOllama() models.ILLMContainerDriver { + return new(ollama) +} + +func (o *ollama) GetType() api.LLMContainerType { + return api.LLM_CONTAINER_OLLAMA +} + +func (o *ollama) GetContainerSpec(ctx context.Context, llm *models.SLLM, image *models.SLLMImage, sku *models.SLLMModel, props []string, devices []computeapi.SIsolatedDevice, diskId string) *computeapi.PodContainerCreateInput { + spec := computeapi.ContainerSpec{ + ContainerSpec: apis.ContainerSpec{ + Image: image.ToContainerImage(), + ImageCredentialId: image.CredentialId, + EnableLxcfs: true, + AlwaysRestart: true, + }, + } + + if len(devices) == 0 && len(*sku.Devices) > 0 { + for i := range *sku.Devices { + index := i + spec.Devices = append(spec.Devices, &computeapi.ContainerDevice{ + Type: apis.CONTAINER_DEVICE_TYPE_ISOLATED_DEVICE, + IsolatedDevice: &computeapi.ContainerIsolatedDevice{ + Index: &index, + }, + }) + } + } else if len(devices) > 0 { + for i := range devices { + spec.Devices = append(spec.Devices, &computeapi.ContainerDevice{ + Type: apis.CONTAINER_DEVICE_TYPE_ISOLATED_DEVICE, + IsolatedDevice: &computeapi.ContainerIsolatedDevice{ + Id: devices[i].Id, + }, + }) + } + } + + // // process rootfs limit + // var diskIndex *int + // if len(diskId) == 0 { + // diskIndex0 := 0 + // diskIndex = &diskIndex0 + // } + // if options.Options.EnableRootfsLimit { + // if !d.RootfsUnlimit { + // spec.RootFs = &apis.ContainerRootfs{ + // Type: apis.CONTAINER_VOLUME_MOUNT_TYPE_DISK, + // Disk: &apis.ContainerVolumeMountDisk{ + // Index: diskIndex, + // Id: diskId, + // SubDirectory: "/rootfs-steam", + // }, + // Persistent: false, + // } + // } + // } + + // process volume mounts + vols := make([]*apis.ContainerVolumeMount, 0) + // appVolIndex := 2 + // postOverlays, err := d.GetMountedAppsPostOverlay() + // if err != nil { + // log.Errorf("GetMountedAppsPostOverlay failed %s", err) + // } + // vols = append(spec.VolumeMounts, GetDiskVolumeMounts(sku.Volumes, appVolIndex, postOverlays)...) + + // udevPath := filepath.Join(GetTmpSocketsHostPath(d.GetName()), "udev") + modelName, modelTag, _ := llm.GetLargeLanguageModelName() + ctrVols := []*apis.ContainerVolumeMount{ + { + UniqueName: "manifests", + Type: apis.CONTAINER_VOLUME_MOUNT_TYPE_HOST_PATH, + MountPath: getManifestsPath(modelName, modelTag), + HostPath: &apis.ContainerVolumeMountHostPath{ + Type: apis.CONTAINER_VOLUME_MOUNT_HOST_PATH_TYPE_FILE, + Path: path.Join(api.LLM_OLLAMA_HOST_PATH, api.LLM_OLLAMA_HOST_MANIFESTS_DIR, modelName+"-"+modelTag), + }, + ReadOnly: true, + }, + { + UniqueName: "blobs", + Type: apis.CONTAINER_VOLUME_MOUNT_TYPE_HOST_PATH, + MountPath: path.Join(api.LLM_OLLAMA_BASE_PATH, api.LLM_OLLAMA_BLOBS_DIR), + HostPath: &apis.ContainerVolumeMountHostPath{ + Type: apis.CONTAINER_VOLUME_MOUNT_HOST_PATH_TYPE_DIRECTORY, + Path: path.Join(api.LLM_OLLAMA_HOST_PATH, api.LLM_OLLAMA_BLOBS_DIR), + }, + ReadOnly: true, + }, + } + vols = append(vols, ctrVols...) + + spec.VolumeMounts = vols + + return &computeapi.PodContainerCreateInput{ + ContainerSpec: spec, + } +} + +// func (o *ollama) PullModelByInstall(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, modelName string, modelTag string) error { +// return nil +// } + +// func (o *ollama) PullModelByGgufFile(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, ggufFileUrl string, model string) error { +// return nil +// } + +// func (o *ollama) GetManifests(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, taskId string) error { +// modelName, modelTag, _ := llm.GetLargeLanguageModelName() +// suffix := fmt.Sprintf("%s/manifests/%s", modelName, modelTag) +// url := fmt.Sprintf(api.LLM_OLLAMA_LIBRARY_BASE_URL, suffix) +// ctr, _ := llm.GetLLMContainer() + +// return download(ctx, userCred, ctr.CmpId, taskId, url, getManifestsPath(modelName, modelTag)) +// } + +// func (o *ollama) AccessBlobsCache(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, taskId string) error { +// ctr, _ := llm.GetLLMContainer() +// modelName, modelTag, _ := llm.GetLargeLanguageModelName() +// blobs, err := fetchBlobs(ctx, userCred, ctr.CmpId, modelName, modelTag) +// if err != nil { +// return errors.Wrapf(err, "failed to fetch blobs for model %s:%s", modelName, modelTag) +// } + +// input := &api.OllamaAccessCacheInput{ +// Blobs: blobs, +// ModelName: modelName, +// } +// _, err = ollama_pod.RequestOllamaBlobsCache(ctx, userCred, ctr.CmpId, taskId, input) + +// return err +// } + +func (o *ollama) CopyBlobs(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM) error { + ctr, _ := llm.GetLLMContainer() + modelName, modelTag, _ := llm.GetLargeLanguageModelName() + blobs, _ := fetchBlobs(ctx, ctr.CmpId, modelName, modelTag) + + blobsTargetDir := path.Join(api.LLM_OLLAMA_BASE_PATH, api.LLM_OLLAMA_BLOBS_DIR) + blobsSrcDir := path.Join(api.LLM_OLLAMA_CACHE_MOUNT_PATH, api.LLM_OLLAMA_CACHE_DIR) + + var commands []string + commands = append(commands, fmt.Sprintf("mkdir -p %s", blobsTargetDir)) + for _, blob := range blobs { + src := path.Join(blobsSrcDir, blob) + target := path.Join(blobsTargetDir, blob) + commands = append(commands, fmt.Sprintf("cp %s %s", src, target)) + } + + cmd := strings.Join(commands, " && ") + if _, err := exec(ctx, ctr.CmpId, "/bin/sh", "-c", cmd); err != nil { + return errors.Wrapf(err, "failed to copy blobs to container") + } + return nil +} + +// func download(ctx context.Context, userCred mcclient.TokenCredential, containerId string, taskId string, webUrl string, path string) error { +// input := &computeapi.ContainerDownloadFileInput{ +// WebUrl: webUrl, +// Path: path, +// } + +// _, err := ollama_pod.RequestDownloadFileIntoContainer(ctx, userCred, containerId, taskId, input) +// return err +// } + +func exec(ctx context.Context, containerId string, command ...string) (string, error) { + // exec command + + input := &computeapi.ContainerExecSyncInput{ + Command: command, + } + resp, err := utils.ExecSyncContainer(ctx, containerId, input) + + // check error and return result + var rst string + if nil != err || resp == nil { + return "", errors.Wrapf(err, "LLM exec error") + } + rst, _ = resp.GetString("stdout") + log.Infoln("llm container exec result: ", resp) + return rst, nil +} + +func getManifestsPath(modelName, modelTag string) string { + return path.Join(api.LLM_OLLAMA_BASE_PATH, api.LLM_OLLAMA_MANIFESTS_BASE_PATH, modelName, modelTag) +} + +func fetchBlobs(ctx context.Context, containerId string, modelName string, modelTag string) ([]string, error) { + manifestContent, err := exec(ctx, containerId, "cat", getManifestsPath(modelName, modelTag)) + if err != nil { + return nil, errors.Wrapf(err, "failed to read manifests from container") + } + + // find all blobs + var results []string + re := regexp.MustCompile(`"digest":"(sha256:[^"]*)"`) + matches := re.FindAllStringSubmatch(manifestContent, -1) + for _, match := range matches { + if len(match) > 1 { + digest := match[1] + processedDigest := strings.Replace(digest, "sha256:", "sha256-", 1) + results = append(results, processedDigest) + } + } + + return results, nil +} diff --git a/pkg/llm/models/access_info.go b/pkg/llm/models/access_info.go new file mode 100644 index 0000000000..271a630afb --- /dev/null +++ b/pkg/llm/models/access_info.go @@ -0,0 +1,48 @@ +package models + +import ( + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/cloudcommon/db" +) + +var accessInfoManager *SAccessInfoManager + +func init() { + GetAccessInfoManager() +} + +func GetAccessInfoManager() *SAccessInfoManager { + if accessInfoManager == nil { + accessInfoManager = &SAccessInfoManager{ + SResourceBaseManager: db.NewResourceBaseManager( + SAccessInfo{}, + "access_infos_tbl", + "access_info", + "access_infos", + ), + } + accessInfoManager.SetVirtualObject(accessInfoManager) + } + return accessInfoManager +} + +type SAccessInfoManager struct { + db.SResourceBaseManager +} + +type SAccessInfo struct { + db.SResourceBase + + LLMId string `width:"128" charset:"ascii" nullable:"true" list:"user" create:"admin_optional" update:"user"` + + // 服务监听端口 + ListenPort int `nullable:"true" create:"optional" list:"user" update:"user"` + // 映射到公网的访问端口 + AccessPort int `nullable:"true" create:"optional" list:"user" update:"user"` + + // 自定义端口类型 + Protocol string `width:"32" charset:"ascii" nullable:"true" list:"user" create:"admin_optional" update:"user"` + + RemoteIps []string `charset:"ascii" nullable:"true" list:"user" create:"admin_optional" update:"user"` + PortMappingEnvs api.PortMappingEnvs `charset:"ascii" nullable:"true" list:"user" create:"admin_optional"` +} diff --git a/pkg/llm/models/dify.go b/pkg/llm/models/dify.go new file mode 100644 index 0000000000..dfb57848a8 --- /dev/null +++ b/pkg/llm/models/dify.go @@ -0,0 +1,292 @@ +package models + +import ( + "context" + "database/sql" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/utils" + "yunion.io/x/sqlchemy" + + commonapi "yunion.io/x/onecloud/pkg/apis" + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/mcclient/modules/compute" +) + +var difyManager *SDifyManager + +func init() { + GetDifyManager() +} + +func GetDifyManager() *SDifyManager { + if difyManager != nil { + return difyManager + } + difyManager = &SDifyManager{ + SLLMBaseManager: NewSLLMBaseManager( + SDify{}, + "difies_tbl", + "dify", + "difies", + ), + } + difyManager.SetVirtualObject(difyManager) + return difyManager +} + +type SDifyManager struct { + SLLMBaseManager +} + +type SDify struct { + SLLMBase + + DifyModelId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"` +} + +func (dm *SDifyManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input *api.DifyCreateInput) (*api.DifyCreateInput, error) { + var err error + input.LLMBaseCreateInput, err = dm.SLLMBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.LLMBaseCreateInput) + if err != nil { + return input, errors.Wrap(err, "validate VirtualResourceCreateInput") + } + model, err := GetDifyModelManager().FetchByIdOrName(ctx, userCred, input.DifyModelId) + if err != nil { + return input, errors.Wrap(err, "fetch DifyModel") + } + dModel := model.(*SDifyModel) + input.DifyModelId = dModel.Id + + return input, nil +} + +func (dm *SDifyManager) OnCreateComplete(ctx context.Context, items []db.IModel, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data []jsonutils.JSONObject) { + parentTaskId, _ := data[0].GetString("parent_task_id") + err := runBatchCreateTask(ctx, items, userCred, data, "DifyBatchCreateTask", parentTaskId) + if err != nil { + for i := range items { + llm := items[i].(*SDify) + llm.SetStatus(ctx, userCred, api.LLM_STATUS_CREATE_FAIL, err.Error()) + } + } +} + +func (dm *SDifyManager) BatchCreateValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input api.DifyCreateInput) (*jsonutils.JSONDict, error) { + data, err := dm.ValidateCreateData(ctx, userCred, ownerId, query, &input) + if err != nil { + return nil, err + } + return data.JSON(data), nil +} + +func (dm *SDifyManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, input api.DifyListInput) (*sqlchemy.SQuery, error) { + q, err := dm.SLLMBaseManager.ListItemFilter(ctx, q, userCred, input.LLMBaseListInput) + if err != nil { + return q, errors.Wrap(err, "VirtualResourceBaseManager.ListItemFilter") + } + if len(input.DifyModel) > 0 { + modelObj, err := GetDifyModelManager().FetchByIdOrName(ctx, userCred, input.DifyModel) + if err != nil { + if errors.Cause(err) == sql.ErrNoRows { + return nil, httperrors.NewResourceNotFoundError2(GetDifyModelManager().KeywordPlural(), input.DifyModel) + } else { + return nil, errors.Wrap(err, "DifyModelManager.FetchByIdOrName") + } + } + q = q.Equals("dify_model_id", modelObj.GetId()) + } + return q, nil +} + +func (dify *SDify) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error { + return dify.StartDeleteTask(ctx, userCred, "") +} + +func (dify *SDify) GetDifyModel(modelId string) (*SDifyModel, error) { + if len(modelId) == 0 { + modelId = dify.DifyModelId + } + model, err := GetDifyModelManager().FetchById(modelId) + if err != nil { + return nil, errors.Wrap(err, "fetch DifyModel") + } + return model.(*SDifyModel), nil +} + +func (dify *SDify) GetDifyContainers() []*computeapi.PodContainerCreateInput { + keys := []string{ + api.DIFY_POSTGRES_KEY, + api.DIFY_REDIS_KEY, + api.DIFY_API_KEY, + api.DIFY_WORKER_KEY, + api.DIFY_WORKER_BEAT_KEY, + api.DIFY_PLUGIN_KEY, + api.DIFY_SANDBOX_KEY, + api.DIFY_SSRF_KEY, + api.DIFY_WEB_KEY, + api.DIFY_NGINX_KEY, + api.DIFY_WEAVIATE_KEY, + } + + var containers []*computeapi.PodContainerCreateInput + for _, key := range keys { + if c, err := dify.getDifyContainerByContainerKey(key); err == nil { + containers = append(containers, c) + } + } + return containers +} + +func (dify *SDify) StartCreateTask(ctx context.Context, userCred mcclient.TokenCredential, input api.DifyCreateInput, parentTaskId string) error { + dify.SetStatus(ctx, userCred, commonapi.STATUS_CREATING, "") + params := jsonutils.Marshal(input).(*jsonutils.JSONDict) + var err = func() error { + task, err := taskman.TaskManager.NewTask(ctx, "DifyCreateTask", dify, userCred, params, parentTaskId, "", nil) + if err != nil { + return errors.Wrapf(err, "NewTask") + } + return task.ScheduleRun(params) + }() + if err != nil { + dify.SetStatus(ctx, userCred, api.LLM_STATUS_CREATE_FAIL, err.Error()) + return err + } + return nil +} + +func (dify *SDify) StartDeleteTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error { + dify.SetStatus(ctx, userCred, api.LLM_STATUS_START_DELETE, "StartDeleteTask") + task, err := taskman.TaskManager.NewTask(ctx, "DifyDeleteTask", dify, userCred, nil, parentTaskId, "", nil) + if err != nil { + return err + } + return task.ScheduleRun(nil) +} + +func (dify *SDify) ServerCreate(ctx context.Context, userCred mcclient.TokenCredential, input *api.DifyCreateInput) (string, error) { + model, err := dify.GetDifyModel(dify.DifyModelId) + if nil != err { + return "", errors.Wrap(err, "GetDifyModel") + } + + // set AutoStart to true + input.AutoStart = true + data, err := GetDifyPodCreateInput(ctx, userCred, input, dify, model, "") + if nil != err { + return "", errors.Wrap(err, "GetDifyPodCreateInput") + } + log.Infoln("PodCreateInput Data: ", jsonutils.Marshal(data).String()) + + s := auth.GetSession(ctx, userCred, "") + resp, err := compute.Servers.Create(s, jsonutils.Marshal(data)) + if nil != err { + return "", errors.Wrap(err, "Servers.Create") + } + id, err := resp.GetString("id") + if nil != err { + return "", errors.Wrap(err, "resp.GetString") + } + + return id, nil +} + +// func (llm *SDify) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error { +// instanceId, isBound, err := llm.IsBoundToInstance() +// if err != nil { +// return errors.Wrap(err, "IsBoundToInstance") +// } +// if isBound { +// return httperrors.NewBadRequestError("llm is bound to instance %s", instanceId) +// } +// return nil +// } + +func (dify *SDify) getDifyContainerByContainerKey(containerKey string) (*computeapi.PodContainerCreateInput, error) { + model, err := dify.GetDifyModel("") + if nil != err { + return nil, err + } + container, err := getDifyContainersManager().GetContainer(dify.GetName(), containerKey, model) + if nil != err { + return nil, err + } + container.AlwaysRestart = true // always restart to solve dependency issue + return container, nil +} + +func (dify *SDify) PerformStart(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + // can't start while it's already running + if utils.IsInStringArray(dify.Status, computeapi.VM_RUNNING_STATUS) { + return nil, errors.Wrapf(errors.ErrInvalidStatus, "dify id: %s status: %s", dify.Id, dify.Status) + } + + if err := dify.StartStartTask(ctx, userCred, ""); err != nil { + return nil, errors.Wrap(err, "StartStartTask") + } + return jsonutils.Marshal(nil), nil +} + +func (dify *SDify) StartStartTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error { + task, err := taskman.TaskManager.NewTask(ctx, "DifyStartTask", dify, userCred, nil, parentTaskId, "", nil) + if err != nil { + return errors.Wrap(err, "NewTask") + } + return task.ScheduleRun(nil) +} + +func (dify *SDify) PerformStop(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + if dify.Status == computeapi.VM_READY { + return nil, errors.Wrapf(errors.ErrInvalidStatus, "dify id: %s status: %s", dify.Id, dify.Status) + } + dify.SetStatus(ctx, userCred, computeapi.VM_START_STOP, "perform stop") + err := dify.StartDifyStopTask(ctx, userCred, "") + if err != nil { + return nil, errors.Wrap(err, "StartStopTask") + } + return nil, nil +} + +func (dify *SDify) StartDifyStopTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error { + task, err := taskman.TaskManager.NewTask(ctx, "DifyStopTask", dify, userCred, nil, parentTaskId, "", nil) + if err != nil { + return errors.Wrap(err, "NewTask") + } + err = task.ScheduleRun(nil) + if err != nil { + return errors.Wrap(err, "ScheduleRun") + } + return nil +} + +// func (dify *SDify) ContainerCreate(ctx context.Context, userCred mcclient.TokenCredential, containerKey string) (string, error) { +// model, err := dify.GetDifyModel("") +// if nil != err { +// return "", errors.Wrap(err, "GetDifyModel") +// } +// // get input +// input, err := getDifyContainersManager().GetContainer(dify.GetName(), containerKey, model) +// if nil != err { +// return "", errors.Wrap(err, "GetContainer") +// } + +// // create on pod +// params := &computeapi.ContainerCreateInput{ +// Spec: input.ContainerSpec, +// } +// s := auth.GetSession(ctx, userCred, "") + +// return "", nil +// } + +func getDifyContainersManager() *DifyContainersManager { + return &DifyContainersManager{} +} diff --git a/pkg/llm/models/dify_container_shared_envs.go b/pkg/llm/models/dify_container_shared_envs.go new file mode 100644 index 0000000000..a00f81c447 --- /dev/null +++ b/pkg/llm/models/dify_container_shared_envs.go @@ -0,0 +1,242 @@ +package models + +import ( + "yunion.io/x/onecloud/pkg/apis" + api "yunion.io/x/onecloud/pkg/apis/llm" +) + +var sharedApiWorkerEnvKeyVal []*apis.ContainerKeyValue + +func getSharedApiWorkerEnv(custom *DifyContainerEnv) []*apis.ContainerKeyValue { + if sharedApiWorkerEnvKeyVal == nil { + sharedApiWorkerEnv := &DifyContainerEnv{ + "CONSOLE_API_URL": "", + "CONSOLE_WEB_URL": "", + "SERVICE_API_URL": "", + "APP_API_URL": "", + "APP_WEB_URL": "", + "LANG": "en_US.UTF-8", + "LC_ALL": "en_US.UTF-8", + "PYTHONIOENCODING": "utf-8", + "LOG_LEVEL": "INFO", + "LOG_FILE": "/app/logs/server.log", + "LOG_FILE_MAX_SIZE": "20", + "LOG_FILE_BACKUP_COUNT": "5", + "LOG_DATEFORMAT": "%Y-%m-%d %H:%M:%S", + "LOG_TZ": "UTC", + "DEBUG": "false", + "FLASK_DEBUG": "false", + "ENABLE_REQUEST_LOGGING": "False", + "SECRET_KEY": api.DIFY_SECRET_KEY, + "INIT_PASSWORD": "", + "DEPLOY_ENV": "PRODUCTION", + "CHECK_UPDATE_URL": "https://updates.dify.ai", + "OPENAI_API_BASE": "https://api.openai.com/v1", + "MIGRATION_ENABLED": "true", + "FILES_ACCESS_TIMEOUT": "300", + "ACCESS_TOKEN_EXPIRE_MINUTES": "60", + "REFRESH_TOKEN_EXPIRE_DAYS": "30", + "APP_MAX_ACTIVE_REQUESTS": "0", + "APP_MAX_EXECUTION_TIME": "1200", + "DIFY_BIND_ADDRESS": "0.0.0.0", + "DIFY_PORT": "5001", + "SERVER_WORKER_AMOUNT": "1", + "SERVER_WORKER_CLASS": "gevent", + "SERVER_WORKER_CONNECTIONS": "10", + "CELERY_WORKER_CLASS": "", + "GUNICORN_TIMEOUT": "360", + "CELERY_WORKER_AMOUNT": "", + "CELERY_AUTO_SCALE": "false", + "CELERY_MAX_WORKERS": "", + "CELERY_MIN_WORKERS": "", + "API_TOOL_DEFAULT_CONNECT_TIMEOUT": "10", + "API_TOOL_DEFAULT_READ_TIMEOUT": "60", + "ENABLE_WEBSITE_JINAREADER": api.DIFY_WEB_ENABLE_WEBSITE_JINAREADER, + "ENABLE_WEBSITE_FIRECRAWL": api.DIFY_WEB_ENABLE_WEBSITE_FIRECRAWL, + "ENABLE_WEBSITE_WATERCRAWL": api.DIFY_WEB_ENABLE_WEBSITE_WATERCRAWL, + "DB_USERNAME": api.DIFY_POSTGRES_USER, + "DB_PASSWORD": api.DIFY_POSTGRES_PASSWORD, + "DB_HOST": api.DIFY_LOCALHOST, + "DB_PORT": api.DIFY_POSTGRES_PORT, + "DB_DATABASE": "dify", + "SQLALCHEMY_POOL_SIZE": "30", + "SQLALCHEMY_POOL_RECYCLE": "3600", + "SQLALCHEMY_ECHO": "false", + "SQLALCHEMY_POOL_PRE_PING": "false", + "SQLALCHEMY_POOL_USE_LIFO": "false", + "POSTGRES_MAX_CONNECTIONS": api.DIFY_POSTGRES_MAX_CONNECTIONS, + "POSTGRES_SHARED_BUFFERS": api.DIFY_POSTGRES_SHARED_BUFFERS, + "POSTGRES_WORK_MEM": api.DIFY_POSTGRES_WORK_MEM, + "POSTGRES_MAINTENANCE_WORK_MEM": api.DIFY_POSTGRES_MAINTENANCE_WORK_MEM, + "POSTGRES_EFFECTIVE_CACHE_SIZE": api.DIFY_POSTGRES_EFFECTIVE_CACHE_SIZE, + "REDIS_HOST": api.DIFY_LOCALHOST, + "REDIS_PORT": api.DIFY_REDIS_PORT, + "REDIS_USERNAME": "", + "REDIS_PASSWORD": api.DIFY_REDISCLI_AUTH, + "REDIS_USE_SSL": "false", + "REDIS_DB": "0", + "REDIS_USE_SENTINEL": "false", + "REDIS_SENTINELS": "", + "REDIS_SENTINEL_SERVICE_NAME": "", + "REDIS_SENTINEL_USERNAME": "", + "REDIS_SENTINEL_PASSWORD": "", + "REDIS_SENTINEL_SOCKET_TIMEOUT": "0.1", + "REDIS_USE_CLUSTERS": "false", + "REDIS_CLUSTERS": "", + "REDIS_CLUSTERS_PASSWORD": "", + "CELERY_BROKER_URL": "redis://:" + api.DIFY_REDISCLI_AUTH + "@" + api.DIFY_LOCALHOST + ":" + api.DIFY_REDIS_PORT + "/1", + "CELERY_BACKEND": api.DIFY_LOCALHOST, + "BROKER_USE_SSL": "false", + "CELERY_USE_SENTINEL": "false", + "CELERY_SENTINEL_MASTER_NAME": "", + "CELERY_SENTINEL_PASSWORD": "", + "CELERY_SENTINEL_SOCKET_TIMEOUT": "0.1", + "STORAGE_TYPE": "opendal", + "OPENDAL_SCHEME": "fs", + "OPENDAL_FS_ROOT": "storage", + "VECTOR_STORE": "weaviate", + "VECTOR_INDEX_NAME_PREFIX": "Vector_index", + "WEAVIATE_ENDPOINT": "http://" + api.DIFY_LOCALHOST + ":8080", + "WEAVIATE_API_KEY": api.DIFY_WEAVIATE_AUTHENTICATION_APIKEY_ALLOWED_KEYS, + "UPLOAD_FILE_SIZE_LIMIT": "15", + "UPLOAD_FILE_BATCH_LIMIT": "5", + "ETL_TYPE": "dify", + "UNSTRUCTURED_API_URL": "", + "UNSTRUCTURED_API_KEY": "", + "SCARF_NO_ANALYTICS": "true", + "PROMPT_GENERATION_MAX_TOKENS": "512", + "CODE_GENERATION_MAX_TOKENS": "1024", + "PLUGIN_BASED_TOKEN_COUNTING_ENABLED": "false", + "MULTIMODAL_SEND_FORMAT": "base64", + "UPLOAD_IMAGE_FILE_SIZE_LIMIT": "10", + "UPLOAD_VIDEO_FILE_SIZE_LIMIT": "100", + "UPLOAD_AUDIO_FILE_SIZE_LIMIT": "50", + "API_SENTRY_DSN": api.DIFY_API_SENTRY_DSN, + "API_SENTRY_TRACES_SAMPLE_RATE": api.DIFY_API_SENTRY_TRACES_SAMPLE_RATE, + "API_SENTRY_PROFILES_SAMPLE_RATE": api.DIFY_API_SENTRY_PROFILES_SAMPLE_RATE, + "WEB_SENTRY_DSN": "", + "NOTION_INTEGRATION_TYPE": "public", + "NOTION_CLIENT_SECRET": "", + "NOTION_CLIENT_ID": "", + "NOTION_INTERNAL_SECRET": "", + "MAIL_TYPE": "resend", + "MAIL_DEFAULT_SEND_FROM": "", + "RESEND_API_URL": "https://api.resend.com", + "RESEND_API_KEY": "your-resend-api-key", + "SMTP_SERVER": "", + "SMTP_PORT": "465", + "SMTP_USERNAME": "", + "SMTP_PASSWORD": "", + "SMTP_USE_TLS": "true", + "SMTP_OPPORTUNISTIC_TLS": "false", + "SENDGRID_API_KEY": "", + "INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH": api.DIFY_WEB_INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH, + "INVITE_EXPIRY_HOURS": "72", + "RESET_PASSWORD_TOKEN_EXPIRY_MINUTES": "5", + "CHANGE_EMAIL_TOKEN_EXPIRY_MINUTES": "5", + "OWNER_TRANSFER_TOKEN_EXPIRY_MINUTES": "5", + "CODE_EXECUTION_ENDPOINT": "http://" + api.DIFY_LOCALHOST + ":8194", + "CODE_EXECUTION_API_KEY": api.DIFY_SANDBOX_API_KEY, + "CODE_MAX_NUMBER": "9223372036854775807", + "CODE_MIN_NUMBER": "-9223372036854775808", + "CODE_MAX_DEPTH": "5", + "CODE_MAX_PRECISION": "20", + "CODE_MAX_STRING_LENGTH": "80000", + "CODE_MAX_STRING_ARRAY_LENGTH": "30", + "CODE_MAX_OBJECT_ARRAY_LENGTH": "30", + "CODE_MAX_NUMBER_ARRAY_LENGTH": "1000", + "CODE_EXECUTION_CONNECT_TIMEOUT": "10", + "CODE_EXECUTION_READ_TIMEOUT": "60", + "CODE_EXECUTION_WRITE_TIMEOUT": "10", + "TEMPLATE_TRANSFORM_MAX_LENGTH": "80000", + "WORKFLOW_MAX_EXECUTION_STEPS": "500", + "WORKFLOW_MAX_EXECUTION_TIME": "1200", + "WORKFLOW_CALL_MAX_DEPTH": "5", + "MAX_VARIABLE_SIZE": "204800", + "WORKFLOW_PARALLEL_DEPTH_LIMIT": "3", + "WORKFLOW_FILE_UPLOAD_LIMIT": "10", + "WORKFLOW_NODE_EXECUTION_STORAGE": "rdbms", + "CORE_WORKFLOW_EXECUTION_REPOSITORY": "core.repositories.sqlalchemy_workflow_execution_repository.SQLAlchemyWorkflowExecutionRepository", + "CORE_WORKFLOW_NODE_EXECUTION_REPOSITORY": "core.repositories.sqlalchemy_workflow_node_execution_repository.SQLAlchemyWorkflowNodeExecutionRepository", + "API_WORKFLOW_NODE_EXECUTION_REPOSITORY": "repositories.sqlalchemy_api_workflow_node_execution_repository.DifyAPISQLAlchemyWorkflowNodeExecutionRepository", + "API_WORKFLOW_RUN_REPOSITORY": "repositories.sqlalchemy_api_workflow_run_repository.DifyAPISQLAlchemyWorkflowRunRepository", + "HTTP_REQUEST_NODE_MAX_BINARY_SIZE": "10485760", + "HTTP_REQUEST_NODE_MAX_TEXT_SIZE": "1048576", + "HTTP_REQUEST_NODE_SSL_VERIFY": "True", + "RESPECT_XFORWARD_HEADERS_ENABLED": "false", + "SSRF_PROXY_HTTP_URL": "http://" + api.DIFY_LOCALHOST + ":3128", + "SSRF_PROXY_HTTPS_URL": "http://" + api.DIFY_LOCALHOST + ":3128", + "LOOP_NODE_MAX_COUNT": api.DIFY_WEB_LOOP_NODE_MAX_COUNT, + "MAX_TOOLS_NUM": api.DIFY_WEB_MAX_TOOLS_NUM, + "MAX_PARALLEL_LIMIT": api.DIFY_WEB_MAX_PARALLEL_LIMIT, + "MAX_ITERATIONS_NUM": api.DIFY_WEB_MAX_ITERATIONS_NUM, + "TEXT_GENERATION_TIMEOUT_MS": api.DIFY_WEB_TEXT_GENERATION_TIMEOUT_MS, + "ALLOW_UNSAFE_DATA_SCHEME": api.DIFY_WEB_ALLOW_UNSAFE_DATA_SCHEME, + "POSTGRES_USER": "${DB_USERNAME}", + "POSTGRES_PASSWORD": "${DB_PASSWORD}", + "POSTGRES_DB": "${DB_DATABASE}", + "PGDATA": "/var/lib/postgresql/data/pgdata", + "SANDBOX_API_KEY": api.DIFY_SANDBOX_API_KEY, + "SANDBOX_GIN_MODE": api.DIFY_SANDBOX_GIN_MODE, + "SANDBOX_WORKER_TIMEOUT": api.DIFY_SANDBOX_WORKER_TIMEOUT, + "SANDBOX_ENABLE_NETWORK": api.DIFY_SANDBOX_ENABLE_NETWORK, + "SANDBOX_HTTP_PROXY": api.DIFY_SANDBOX_HTTP_PROXY, + "SANDBOX_HTTPS_PROXY": api.DIFY_SANDBOX_HTTPS_PROXY, + "SANDBOX_PORT": api.DIFY_SANDBOX_PORT, + "WEAVIATE_PERSISTENCE_DATA_PATH": api.DIFY_WEAVIATE_PERSISTENCE_DATA_PATH, + "WEAVIATE_QUERY_DEFAULTS_LIMIT": api.DIFY_WEAVIATE_QUERY_DEFAULTS_LIMIT, + "WEAVIATE_AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED": api.DIFY_WEAVIATE_AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED, + "WEAVIATE_DEFAULT_VECTORIZER_MODULE": api.DIFY_WEAVIATE_DEFAULT_VECTORIZER_MODULE, + "WEAVIATE_CLUSTER_HOSTNAME": api.DIFY_WEAVIATE_CLUSTER_HOSTNAME, + "WEAVIATE_AUTHENTICATION_APIKEY_ENABLED": api.DIFY_WEAVIATE_AUTHENTICATION_APIKEY_ENABLED, + "WEAVIATE_AUTHENTICATION_APIKEY_ALLOWED_KEYS": api.DIFY_WEAVIATE_AUTHENTICATION_APIKEY_ALLOWED_KEYS, + "WEAVIATE_AUTHENTICATION_APIKEY_USERS": api.DIFY_WEAVIATE_AUTHENTICATION_APIKEY_USERS, + "WEAVIATE_AUTHORIZATION_ADMINLIST_ENABLED": api.DIFY_WEAVIATE_AUTHORIZATION_ADMINLIST_ENABLED, + "WEAVIATE_AUTHORIZATION_ADMINLIST_USERS": api.DIFY_WEAVIATE_AUTHORIZATION_ADMINLIST_USERS, + "NGINX_SERVER_NAME": api.DIFY_NGINX_SERVER_NAME, + "NGINX_PORT": api.DIFY_NGINX_PORT, + "NGINX_WORKER_PROCESSES": api.DIFY_NGINX_WORKER_PROCESSES, + "NGINX_CLIENT_MAX_BODY_SIZE": api.DIFY_NGINX_CLIENT_MAX_BODY_SIZE, + "NGINX_KEEPALIVE_TIMEOUT": api.DIFY_NGINX_KEEPALIVE_TIMEOUT, + "NGINX_PROXY_READ_TIMEOUT": api.DIFY_NGINX_PROXY_READ_TIMEOUT, + "NGINX_PROXY_SEND_TIMEOUT": api.DIFY_NGINX_PROXY_SEND_TIMEOUT, + "SSRF_HTTP_PORT": api.DIFY_SSRF_HTTP_PORT, + "SSRF_COREDUMP_DIR": api.DIFY_SSRF_COREDUMP_DIR, + "SSRF_REVERSE_PROXY_PORT": api.DIFY_SANDBOX_PORT, + "SSRF_SANDBOX_HOST": api.DIFY_LOCALHOST, + "SSRF_DEFAULT_TIME_OUT": "5", + "SSRF_DEFAULT_CONNECT_TIME_OUT": "5", + "SSRF_DEFAULT_READ_TIME_OUT": "5", + "SSRF_DEFAULT_WRITE_TIME_OUT": "5", + "DB_PLUGIN_DATABASE": api.DIFY_PLUGIN_DB_DATABASE, + "PLUGIN_DAEMON_PORT": api.DIFY_PLUGIN_SERVER_PORT, + "PLUGIN_DAEMON_KEY": api.DIFY_PLUGIN_SERVER_KEY, + "PLUGIN_DAEMON_URL": "http://" + api.DIFY_LOCALHOST + ":" + api.DIFY_PLUGIN_SERVER_PORT, + "PLUGIN_MAX_PACKAGE_SIZE": api.DIFY_PLUGIN_MAX_PACKAGE_SIZE, + "PLUGIN_PPROF_ENABLED": api.DIFY_PLUGIN_PPROF_ENABLED, + "PLUGIN_DEBUGGING_HOST": api.DIFY_PLUGIN_REMOTE_INSTALLING_HOST, + "PLUGIN_DEBUGGING_PORT": api.DIFY_PLUGIN_REMOTE_INSTALLING_PORT, + "EXPOSE_PLUGIN_DEBUGGING_HOST": api.DIFY_LOCALHOST, + "EXPOSE_PLUGIN_DEBUGGING_PORT": api.DIFY_PLUGIN_REMOTE_INSTALLING_PORT, + "PLUGIN_DIFY_INNER_API_KEY": api.DIFY_API_INNER_KEY, + "PLUGIN_DIFY_INNER_API_URL": "http://" + api.DIFY_LOCALHOST + ":5001", + "ENDPOINT_URL_TEMPLATE": "http://" + api.DIFY_LOCALHOST + "/e/{hook_id}", + "MARKETPLACE_ENABLED": "true", + "MARKETPLACE_API_URL": api.DIFY_WEB_MARKETPLACE_API_URL, + "ALLOW_EMBED": api.DIFY_WEB_ALLOW_EMBED, + "QUEUE_MONITOR_THRESHOLD": "200", + "QUEUE_MONITOR_ALERT_EMAILS": "", + "QUEUE_MONITOR_INTERVAL": "30", + "ENABLE_CLEAN_EMBEDDING_CACHE_TASK": "false", + "ENABLE_CLEAN_UNUSED_DATASETS_TASK": "false", + "ENABLE_CREATE_TIDB_SERVERLESS_TASK": "false", + "ENABLE_UPDATE_TIDB_SERVERLESS_STATUS_TASK": "false", + "ENABLE_CLEAN_MESSAGES": "false", + "ENABLE_MAIL_CLEAN_DOCUMENT_NOTIFY_TASK": "false", + "ENABLE_DATASETS_QUEUE_MONITOR": "false", + "ENABLE_CHECK_UPGRADABLE_PLUGIN_TASK": "true", + } + sharedApiWorkerEnvKeyVal = sharedApiWorkerEnv.GetContainerEnvs(custom) + } + return sharedApiWorkerEnvKeyVal +} diff --git a/pkg/llm/models/dify_containers.go b/pkg/llm/models/dify_containers.go new file mode 100644 index 0000000000..6340480f38 --- /dev/null +++ b/pkg/llm/models/dify_containers.go @@ -0,0 +1,442 @@ +package models + +import ( + "errors" + "fmt" + "path" + + "yunion.io/x/onecloud/pkg/apis" + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + api "yunion.io/x/onecloud/pkg/apis/llm" +) + +type DifyContainerEnv map[string]string + +func (envsPtr *DifyContainerEnv) GetContainerEnvs(userCustomizedEnvs *DifyContainerEnv) []*apis.ContainerKeyValue { + envs := *envsPtr + if len(envs) == 0 { + return nil + } + + ctrEnvs := make([]*apis.ContainerKeyValue, 0, len(envs)) + for key, value := range envs { + if userCustomizedEnvs != nil && (*userCustomizedEnvs)[key] != "" { + value = (*userCustomizedEnvs)[key] + } + ctrEnvs = append(ctrEnvs, &apis.ContainerKeyValue{ + Key: key, + Value: value, + }) + } + return ctrEnvs +} + +func (envsPtr *DifyContainerEnv) SetContainerEnv(key, value string) error { + if key == "" { + return errors.New("environment variable key cannot be empty") + } + + if envsPtr == nil { + return errors.New("DifyContainerEnv pointer is nil") + } + + if *envsPtr == nil { + *envsPtr = make(DifyContainerEnv) + } + + (*envsPtr)[key] = value + return nil +} + +func getPVCMount(name, subDir, mountPath string) *apis.ContainerVolumeMount { + diskIndex := 0 + pvc := &apis.ContainerVolumeMount{ + UniqueName: name + "-data-pvc", + Type: apis.CONTAINER_VOLUME_MOUNT_TYPE_DISK, + Disk: &apis.ContainerVolumeMountDisk{ + SubDirectory: subDir, + Index: &diskIndex, + }, + MountPath: mountPath, + ReadOnly: false, + Propagation: apis.MOUNTPROPAGATION_PROPAGATION_PRIVATE, + } + + return pvc +} + +type DifyContainersManager struct { + UserCustomizedEnvs *DifyContainerEnv +} + +func _getRegistryImage(imageId string) string { + image, err := GetLLMImageManager().FetchById(imageId) + if err != nil { + return "" + } + return image.(*SLLMImage).ToContainerImage() +} + +func (m *DifyContainersManager) GetContainer(name, containerKey string, sku *SDifyModel) (*computeapi.PodContainerCreateInput, error) { + switch containerKey { + case api.DIFY_REDIS_KEY: + return m._getRedisContainer(name, containerKey, _getRegistryImage(sku.RedisImageId)), nil + case api.DIFY_POSTGRES_KEY: + return m._getPostgresContainer(name, containerKey, _getRegistryImage(sku.PostgresImageId)), nil + case api.DIFY_API_KEY: + return m._getApiContainer(name, containerKey, _getRegistryImage(sku.DifyApiImageId)), nil + case api.DIFY_WORKER_KEY: + return m._getWorkerContainer(name, containerKey, _getRegistryImage(sku.DifyApiImageId)), nil + case api.DIFY_WORKER_BEAT_KEY: + return m._getWorkerBeatContainer(name, containerKey, _getRegistryImage(sku.DifyApiImageId)), nil + case api.DIFY_PLUGIN_KEY: + return m._getPluginContainer(name, containerKey, _getRegistryImage(sku.DifyPluginImageId)), nil + case api.DIFY_WEB_KEY: + return m._getWebContainer(name, containerKey, _getRegistryImage(sku.DifyWebImageId)), nil + case api.DIFY_SSRF_KEY: + return m._getSsrfContainer(name, containerKey, _getRegistryImage(sku.DifySSRFImageId)), nil + case api.DIFY_NGINX_KEY: + return m._getNginxContainer(name, containerKey, _getRegistryImage(sku.NginxImageId)), nil + case api.DIFY_WEAVIATE_KEY: + return m._getWeaviateContainer(name, containerKey, _getRegistryImage(sku.DifyWeaviateImageId)), nil + case api.DIFY_SANDBOX_KEY: + return m._getSandboxContainer(name, containerKey, _getRegistryImage(sku.DifySandboxImageId)), nil + default: + return nil, errors.New("unsupported container key") + } +} + +func (m *DifyContainersManager) _getRedisContainer(name, key, image string) *computeapi.PodContainerCreateInput { + // set name and image + ctr := &computeapi.PodContainerCreateInput{ + Name: name + "-" + key, + } + ctr.Image = image + + // set container environments + envs := &DifyContainerEnv{ + "REDISCLI_AUTH": api.DIFY_REDISCLI_AUTH, + } + ctr.Envs = envs.GetContainerEnvs(m.UserCustomizedEnvs) + + // set PVC to store data + ctr.VolumeMounts = []*apis.ContainerVolumeMount{ + getPVCMount(key, key, api.DIFY_REDIS_PVC_MOUNT_PATH), + } + + // set command + ctr.Args = []string{ + "redis-server", "--requirepass", api.DIFY_REDISCLI_AUTH, + } + + return ctr +} + +func (m *DifyContainersManager) _getPostgresContainer(name, key, image string) *computeapi.PodContainerCreateInput { + // set name and image + ctr := &computeapi.PodContainerCreateInput{ + Name: name + "-" + key, + } + ctr.Image = image + + // set container environments + envs := &DifyContainerEnv{ + "POSTGRES_USER": api.DIFY_POSTGRES_USER, + "POSTGRES_PASSWORD": api.DIFY_POSTGRES_PASSWORD, + "POSTGRES_DB": api.DIFY_POSTGRES_DB, + "PGDATA": path.Join(api.DIFY_POSTGRES_PVC_MOUNT_PATH, api.DIFY_POSTGRES_PGDATA), + } + ctr.Envs = envs.GetContainerEnvs(m.UserCustomizedEnvs) + + // set PVC to store data + ctr.VolumeMounts = []*apis.ContainerVolumeMount{ + getPVCMount(key, key, api.DIFY_POSTGRES_PVC_MOUNT_PATH), + } + + // set command for ctr + ctr.Args = []string{ + "postgres", + "-c", "max_connections=" + api.DIFY_POSTGRES_MAX_CONNECTIONS, + "-c", "shared_buffers=" + api.DIFY_POSTGRES_SHARED_BUFFERS, + "-c", "work_mem=" + api.DIFY_POSTGRES_WORK_MEM, + "-c", "maintenance_work_mem=" + api.DIFY_POSTGRES_MAINTENANCE_WORK_MEM, + "-c", "effective_cache_size=" + api.DIFY_POSTGRES_EFFECTIVE_CACHE_SIZE, + } + + return ctr +} + +func (m *DifyContainersManager) _getApiContainer(name, key, image string) *computeapi.PodContainerCreateInput { + // set name and image + ctr := &computeapi.PodContainerCreateInput{ + Name: name + "-" + key, + } + ctr.Image = image + + // set container environments + envs := &DifyContainerEnv{ + "MODE": api.DIFY_API_MODE, + "SENTRY_DSN": api.DIFY_API_SENTRY_DSN, + "SENTRY_TRACES_SAMPLE_RATE": api.DIFY_API_SENTRY_TRACES_SAMPLE_RATE, + "SENTRY_PROFILES_SAMPLE_RATE": api.DIFY_API_SENTRY_PROFILES_SAMPLE_RATE, + "PLUGIN_REMOTE_INSTALL_HOST": api.DIFY_LOCALHOST, + "PLUGIN_REMOTE_INSTALL_PORT": api.DIFY_PLUGIN_REMOTE_INSTALLING_PORT, + "PLUGIN_MAX_PACKAGE_SIZE": api.DIFY_PLUGIN_MAX_PACKAGE_SIZE, + "INNER_API_KEY_FOR_PLUGIN": api.DIFY_API_INNER_KEY, + } + ctr.Envs = append(getSharedApiWorkerEnv(m.UserCustomizedEnvs), envs.GetContainerEnvs(m.UserCustomizedEnvs)...) + + // set PVC to store data + ctr.VolumeMounts = []*apis.ContainerVolumeMount{ + getPVCMount(key, "api", api.DIFY_API_PVC_MOUNT_PATH), + } + + return ctr +} + +func (m *DifyContainersManager) _getWorkerContainer(name, key, image string) *computeapi.PodContainerCreateInput { + // set name and image + ctr := &computeapi.PodContainerCreateInput{ + Name: name + "-" + key, + } + ctr.Image = image + + // set container environments + envs := &DifyContainerEnv{ + "MODE": api.DIFY_WORKER_MODE, + "SENTRY_DSN": api.DIFY_API_SENTRY_DSN, + "SENTRY_TRACES_SAMPLE_RATE": api.DIFY_API_SENTRY_TRACES_SAMPLE_RATE, + "SENTRY_PROFILES_SAMPLE_RATE": api.DIFY_API_SENTRY_PROFILES_SAMPLE_RATE, + "PLUGIN_MAX_PACKAGE_SIZE": api.DIFY_PLUGIN_MAX_PACKAGE_SIZE, + "INNER_API_KEY_FOR_PLUGIN": api.DIFY_API_INNER_KEY, + } + ctr.Envs = append(getSharedApiWorkerEnv(m.UserCustomizedEnvs), envs.GetContainerEnvs(m.UserCustomizedEnvs)...) + + // set PVC to store data + ctr.VolumeMounts = []*apis.ContainerVolumeMount{ + getPVCMount(key, "api", api.DIFY_API_PVC_MOUNT_PATH), + } + + return ctr +} + +func (m *DifyContainersManager) _getWorkerBeatContainer(name, key, image string) *computeapi.PodContainerCreateInput { + // set name and image + ctr := &computeapi.PodContainerCreateInput{ + Name: name + "-" + key, + } + ctr.Image = image + + // set container environments + envs := &DifyContainerEnv{ + "MODE": api.DIFY_WORKER_BEAT_MODE, + } + ctr.Envs = append(getSharedApiWorkerEnv(m.UserCustomizedEnvs), envs.GetContainerEnvs(m.UserCustomizedEnvs)...) + + return ctr +} + +func (m *DifyContainersManager) _getPluginContainer(name, key, image string) *computeapi.PodContainerCreateInput { + // set name and image + ctr := &computeapi.PodContainerCreateInput{ + Name: name + "-" + key, + } + ctr.Image = image + + // set container environments + envs := &DifyContainerEnv{ + "DB_DATABASE": api.DIFY_PLUGIN_DB_DATABASE, + "SERVER_PORT": api.DIFY_PLUGIN_SERVER_PORT, + "SERVER_KEY": api.DIFY_PLUGIN_SERVER_KEY, + "MAX_PACKAGE_CACHE_PATH": api.DIFY_PLUGIN_PACKAGE_CACHE_PATH, + "PPROF_ENABLED": api.DIFY_PLUGIN_PPROF_ENABLED, + "DIFY_INNER_API_URL": api.DIFY_PLUGIN_DIFY_INNER_API_URL, + "DIFY_INNER_API_KEY": api.DIFY_API_INNER_KEY, + "PLUGIN_REMOTE_INSTALLING_HOST": api.DIFY_PLUGIN_REMOTE_INSTALLING_HOST, + "PLUGIN_REMOTE_INSTALLING_PORT": api.DIFY_PLUGIN_REMOTE_INSTALLING_PORT, + "PLUGIN_WORKING_PATH": api.DIFY_PLUGIN_WORKING_PATH, + "FORCE_VERIFYING_SIGNATURE": api.DIFY_PLUGIN_FORCE_VERIFYING_SIGNATURE, + "PYTHON_ENV_INIT_TIMEOUT": api.DIFY_PLUGIN_PYTHON_ENV_INIT_TIMEOUT, + "PLUGIN_MAX_EXECUTION_TIMEOUT": api.DIFY_PLUGIN_MAX_EXECUTION_TIMEOUT, + "PIP_MIRROR_URL": api.PIP_MIRROR_URL, + "PLUGIN_STORAGE_TYPE": api.DIFY_PLUGIN_STORAGE_TYPE, + "PLUGIN_STORAGE_LOCAL_ROOT": api.DIFY_PLUGIN_STORAGE_LOCAL_ROOT, + "PLUGIN_INSTALLED_PATH": api.DIFY_PLUGIN_INSTALLED_PATH, + "PLUGIN_PACKAGE_CACHE_PATH": api.DIFY_PLUGIN_PACKAGE_CACHE_PATH, + "PLUGIN_MEDIA_CACHE_PATH": api.DIFY_PLUGIN_MEDIA_CACHE_PATH, + } + ctr.Envs = append(getSharedApiWorkerEnv(m.UserCustomizedEnvs), envs.GetContainerEnvs(m.UserCustomizedEnvs)...) + + // set PVC to store data + ctr.VolumeMounts = []*apis.ContainerVolumeMount{ + getPVCMount(key, key, api.DIFY_PLUGIN_STORAGE_LOCAL_ROOT), + } + + return ctr +} + +func (m *DifyContainersManager) _getWebContainer(name, key, image string) *computeapi.PodContainerCreateInput { + // set name and image + ctr := &computeapi.PodContainerCreateInput{ + Name: name + "-" + key, + } + ctr.Image = image + + // set container environments + envs := &DifyContainerEnv{ + "HOSTNAME": "", // set HOSTNAME to empty, to avoid Error: getaddrinfo ENOTFOUND + "CONSOLE_API_URL": api.DIFY_WEB_CONSOLE_API_URL, + "APP_API_URL": api.DIFY_WEB_APP_API_URL, + "SENTRY_DSN": api.DIFY_WEB_SENTRY_DSN, + "NEXT_TELEMETRY_DISABLED": api.DIFY_WEB_NEXT_TELEMETRY_DISABLED, + "TEXT_GENERATION_TIMEOUT_MS": api.DIFY_WEB_TEXT_GENERATION_TIMEOUT_MS, + "CSP_WHITELIST": api.DIFY_WEB_CSP_WHITELIST, + "ALLOW_EMBED": api.DIFY_WEB_ALLOW_EMBED, + "ALLOW_UNSAFE_DATA_SCHEME": api.DIFY_WEB_ALLOW_UNSAFE_DATA_SCHEME, + "MARKETPLACE_API_URL": api.DIFY_WEB_MARKETPLACE_API_URL, + "MARKETPLACE_URL": api.DIFY_WEB_MARKETPLACE_URL, + "TOP_K_MAX_VALUE": api.DIFY_WEB_TOP_K_MAX_VALUE, + "INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH": api.DIFY_WEB_INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH, + "PM2_INSTANCES": api.DIFY_WEB_PM2_INSTANCES, + "LOOP_NODE_MAX_COUNT": api.DIFY_WEB_LOOP_NODE_MAX_COUNT, + "MAX_TOOLS_NUM": api.DIFY_WEB_MAX_TOOLS_NUM, + "MAX_PARALLEL_LIMIT": api.DIFY_WEB_MAX_PARALLEL_LIMIT, + "MAX_ITERATIONS_NUM": api.DIFY_WEB_MAX_ITERATIONS_NUM, + "ENABLE_WEBSITE_JINAREADER": api.DIFY_WEB_ENABLE_WEBSITE_JINAREADER, + "ENABLE_WEBSITE_FIRECRAWL": api.DIFY_WEB_ENABLE_WEBSITE_FIRECRAWL, + "ENABLE_WEBSITE_WATERCRAWL": api.DIFY_WEB_ENABLE_WEBSITE_WATERCRAWL, + } + ctr.Envs = envs.GetContainerEnvs(m.UserCustomizedEnvs) + + return ctr +} + +func (m *DifyContainersManager) _getSsrfContainer(name, key, image string) *computeapi.PodContainerCreateInput { + // set name and image + ctr := &computeapi.PodContainerCreateInput{ + Name: name + "-" + key, + } + ctr.Image = image + + // set container environments + envs := &DifyContainerEnv{ + // "VISIBLE_HOSTNAME": "localhost", // set VISIBLE_HOSTNAME to localhost, to avoid rDNS test failed + "HTTP_PORT": api.DIFY_SSRF_HTTP_PORT, + "COREDUMP_DIR": api.DIFY_SSRF_COREDUMP_DIR, + "REVERSE_PROXY_PORT": api.DIFY_SANDBOX_PORT, + "SANDBOX_HOST": api.DIFY_LOCALHOST, + "SANDBOX_PORT": api.DIFY_SANDBOX_PORT, + } + ctr.Envs = envs.GetContainerEnvs(m.UserCustomizedEnvs) + + // set PVC to store data + // ctr.VolumeMounts = []*apis.ContainerVolumeMount{ + // getPVCMount(key, key, api.SSRF_MOUNT_PATH), + // } + + // generate entrypoint + entrypointSH := fmt.Sprintf(api.DIFY_SSRF_ENTRYPINT_SHELL, api.DIFY_SSRF_SQUID_CONFIGURATION_FILE) + ctr.Command = []string{ + "/bin/sh", "-c", entrypointSH, + } + + return ctr +} + +func (m *DifyContainersManager) _getNginxContainer(name, key, image string) *computeapi.PodContainerCreateInput { + // set name and image + ctr := &computeapi.PodContainerCreateInput{ + Name: name + "-" + key, + } + ctr.Image = image + + // set container environments + envs := &DifyContainerEnv{ + "NGINX_SERVER_NAME": api.DIFY_NGINX_SERVER_NAME, + "NGINX_PORT": api.DIFY_NGINX_PORT, + "NGINX_WORKER_PROCESSES": api.DIFY_NGINX_WORKER_PROCESSES, + "NGINX_CLIENT_MAX_BODY_SIZE": api.DIFY_NGINX_CLIENT_MAX_BODY_SIZE, + "NGINX_KEEPALIVE_TIMEOUT": api.DIFY_NGINX_KEEPALIVE_TIMEOUT, + "NGINX_PROXY_READ_TIMEOUT": api.DIFY_NGINX_PROXY_READ_TIMEOUT, + "NGINX_PROXY_SEND_TIMEOUT": api.DIFY_NGINX_PROXY_SEND_TIMEOUT, + } + ctr.Envs = envs.GetContainerEnvs(m.UserCustomizedEnvs) + + // set PVC to store data + ctr.VolumeMounts = []*apis.ContainerVolumeMount{ + getPVCMount(key, key, api.DIFY_NGINX_MOUNT_PATH), + } + + // generate entrypoint + entrypointSH := fmt.Sprintf(api.DIFY_NGINX_ENTRYPINT_SHELL, api.DIFY_NGINX_NGINX_CONF_FILE, api.DIFY_NGINX_PROXY_CONF_FILE, api.DIFY_NGINX_DEFAULT_CONF_FILE) + ctr.Command = []string{ + "/bin/sh", "-c", entrypointSH, + } + + return ctr +} + +func (m *DifyContainersManager) _getWeaviateContainer(name, key, image string) *computeapi.PodContainerCreateInput { + // set name and image + ctr := &computeapi.PodContainerCreateInput{ + Name: name + "-" + key, + } + ctr.Image = image + + // set container environments + envs := &DifyContainerEnv{ + "PERSISTENCE_DATA_PATH": api.DIFY_WEAVIATE_PERSISTENCE_DATA_PATH, + "QUERY_DEFAULTS_LIMIT": api.DIFY_WEAVIATE_QUERY_DEFAULTS_LIMIT, + "AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED": api.DIFY_WEAVIATE_AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED, + "DEFAULT_VECTORIZER_MODULE": api.DIFY_WEAVIATE_DEFAULT_VECTORIZER_MODULE, + "CLUSTER_HOSTNAME": api.DIFY_WEAVIATE_CLUSTER_HOSTNAME, + "AUTHENTICATION_APIKEY_ENABLED": api.DIFY_WEAVIATE_AUTHENTICATION_APIKEY_ENABLED, + "AUTHENTICATION_APIKEY_ALLOWED_KEYS": api.DIFY_WEAVIATE_AUTHENTICATION_APIKEY_ALLOWED_KEYS, + "AUTHENTICATION_APIKEY_USERS": api.DIFY_WEAVIATE_AUTHENTICATION_APIKEY_USERS, + "AUTHORIZATION_ADMINLIST_ENABLED": api.DIFY_WEAVIATE_AUTHORIZATION_ADMINLIST_ENABLED, + "AUTHORIZATION_ADMINLIST_USERS": api.DIFY_WEAVIATE_AUTHORIZATION_ADMINLIST_USERS, + } + ctr.Envs = envs.GetContainerEnvs(m.UserCustomizedEnvs) + + // set PVC to store data + ctr.VolumeMounts = []*apis.ContainerVolumeMount{ + getPVCMount(key, key, api.DIFY_WEAVIATE_PERSISTENCE_DATA_PATH), + } + + return ctr +} + +func (m *DifyContainersManager) _getSandboxContainer(name, key, image string) *computeapi.PodContainerCreateInput { + // set name and image + ctr := &computeapi.PodContainerCreateInput{ + Name: name + "-" + key, + } + ctr.Image = image + + // set container environments + envs := &DifyContainerEnv{ + "API_KEY": api.DIFY_SANDBOX_API_KEY, + "GIN_MODE": api.DIFY_SANDBOX_GIN_MODE, + "WORKER_TIMEOUT": api.DIFY_SANDBOX_WORKER_TIMEOUT, + "ENABLE_NETWORK": api.DIFY_SANDBOX_ENABLE_NETWORK, + "HTTP_PROXY": api.DIFY_SANDBOX_HTTP_PROXY, + "HTTPS_PROXY": api.DIFY_SANDBOX_HTTPS_PROXY, + "SANDBOX_PORT": api.DIFY_SANDBOX_PORT, + "PIP_MIRROR_URL": api.PIP_MIRROR_URL, + } + ctr.Envs = envs.GetContainerEnvs(m.UserCustomizedEnvs) + + // set PVC to store data + ctr.VolumeMounts = []*apis.ContainerVolumeMount{ + getPVCMount(key+"conf", key+"conf", api.DIFY_SANDBOX_CONF_MOUNT_PATH), + getPVCMount(key+"dep", key+"dep", api.DIFY_SANDBOX_DEP_MOUNT_PATH), + } + + // set command + writeConfigCommand := fmt.Sprintf(api.DIFY_SANDBOX_WRITE_CONF_SHELL, api.DIFY_SANDBOX_CONF_FILE, api.DIFY_SANDBOX_CONF_TEMP_FILE) + ctr.Command = []string{ + "/bin/sh", "-c", writeConfigCommand, + } + + return ctr +} diff --git a/pkg/llm/models/dify_model.go b/pkg/llm/models/dify_model.go new file mode 100644 index 0000000000..baaad9b740 --- /dev/null +++ b/pkg/llm/models/dify_model.go @@ -0,0 +1,125 @@ +package models + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + "yunion.io/x/sqlchemy" + + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/cloudcommon/validators" + "yunion.io/x/onecloud/pkg/mcclient" +) + +func init() { + GetDifyModelManager() +} + +var difyModelManager *SDifyModelManager + +func GetDifyModelManager() *SDifyModelManager { + if difyModelManager != nil { + return difyModelManager + } + difyModelManager = &SDifyModelManager{ + SLLMModelBaseManager: NewSLLMModelBaseManager( + SDifyModel{}, + "dify_models_tbl", + "dify_model", + "dify_models", + ), + } + difyModelManager.SetVirtualObject(difyModelManager) + return difyModelManager +} + +type SDifyModelManager struct { + SLLMModelBaseManager +} + +type SDifyModel struct { + SLLMModelBase + + PostgresImageId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"` + RedisImageId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"` + NginxImageId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"` + DifyApiImageId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"` + DifyPluginImageId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"` + DifyWebImageId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"` + DifySandboxImageId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"` + DifySSRFImageId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"` + DifyWeaviateImageId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"` +} + +func (man *SDifyModelManager) ListItemFilter( + ctx context.Context, + q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, + input api.DifyModelListInput, +) (*sqlchemy.SQuery, error) { + var err error + q, err = man.SLLMModelBaseManager.ListItemFilter(ctx, q, userCred, input.SharableVirtualResourceListInput) + if err != nil { + return nil, errors.Wrapf(err, "SLLMModelBaseManager.ListItemFilter") + } + return q, nil +} + +// func (man *SDifyModelManager) FetchCustomizeColumns( +// ctx context.Context, +// userCred mcclient.TokenCredential, +// query jsonutils.JSONObject, +// objs []interface{}, +// fields stringutils2.SSortedStrings, +// isList bool, +// ) []api.LLMModelDetails { + +// } + +func (man *SDifyModelManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input *api.DifyModelCreateInput) (*api.DifyModelCreateInput, error) { + var err error + input.LLMModelBaseCreateInput, err = man.SLLMModelBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.LLMModelBaseCreateInput) + if err != nil { + return nil, errors.Wrap(err, "SLLMModelBaseManager.ValidateCreateData") + } + + for _, imgId := range []*string{&input.PostgresImageId, &input.RedisImageId, &input.NginxImageId, &input.DifyApiImageId, &input.DifyPluginImageId, &input.DifyWebImageId, &input.DifySandboxImageId, &input.DifySSRFImageId, &input.DifyWeaviateImageId} { + _, err := validators.ValidateModel(ctx, userCred, GetLLMImageManager(), imgId) + if err != nil { + return input, errors.Wrapf(err, "validate image_id %s", *imgId) + } + } + + return input, nil +} + +func (model *SDifyModel) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.DifyModelUpdateInput) (api.DifyModelUpdateInput, error) { + var err error + input.LLMModelBaseUpdateInput, err = model.SLLMModelBase.ValidateUpdateData(ctx, userCred, query, input.LLMModelBaseUpdateInput) + if err != nil { + return input, errors.Wrap(err, "validate LLMModelBaseUpdateInput") + } + + for _, imgId := range []*string{&input.PostgresImageId, &input.RedisImageId, &input.NginxImageId, &input.DifyApiImageId, &input.DifyPluginImageId, &input.DifyWebImageId, &input.DifySandboxImageId, &input.DifySSRFImageId, &input.DifyWeaviateImageId} { + if *imgId != "" { + _, err := validators.ValidateModel(ctx, userCred, GetLLMImageManager(), imgId) + if err != nil { + return input, errors.Wrapf(err, "validate image_id %s", *imgId) + } + } + } + + return input, nil +} + +func (model *SDifyModel) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error { + count, err := GetDifyManager().Query().Equals("dify_model_id", model.Id).CountWithError() + if nil != err { + return errors.Wrap(err, "fetch dify") + } + if count > 0 { + return errors.Wrap(errors.ErrNotSupported, "This model is currently in use") + } + return nil +} diff --git a/pkg/llm/models/dify_pod.go b/pkg/llm/models/dify_pod.go new file mode 100644 index 0000000000..cb8303a51a --- /dev/null +++ b/pkg/llm/models/dify_pod.go @@ -0,0 +1,130 @@ +package models + +import ( + "context" + + "yunion.io/x/jsonutils" + + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/mcclient" +) + +func GetDifyPodCreateInput( + ctx context.Context, + userCred mcclient.TokenCredential, + input *api.DifyCreateInput, + dify *SDify, + sku *SDifyModel, + eip string, +) (*computeapi.ServerCreateInput, error) { + data := computeapi.ServerCreateInput{} + data.AutoStart = input.AutoStart + data.ServerConfigs = computeapi.NewServerConfigs() + data.Hypervisor = computeapi.HYPERVISOR_POD + + postStopCleanupConfgi := PodPostStopCleanupConfig{ + Dirs: []string{ + GetTmpHostPath(dify.GetName()), + }, + } + data.Metadata = map[string]string{ + POD_METADATA_POST_STOP_CLEANUP_CONFIG: jsonutils.Marshal(postStopCleanupConfgi).String(), + } + + data.VcpuCount = sku.Cpu + data.VmemSize = sku.Memory + 1 + data.Name = input.Name + + // disks + data.Disks = make([]*computeapi.DiskConfig, 0) + if sku.Volumes != nil && !sku.Volumes.IsZero() { + for idx, volume := range *sku.Volumes { + data.Disks = append(data.Disks, &computeapi.DiskConfig{ + DiskType: "data", + Format: "raw", + Fs: "ext4", + SizeMb: volume.SizeMB, + Index: idx, + }) + } + } + + // isolated devices + if sku.Devices != nil && !sku.Devices.IsZero() { + data.IsolatedDevices = make([]*computeapi.IsolatedDeviceConfig, 0) + devices := *sku.Devices + for i := 0; i < len(devices); i++ { + isolatedDevice := &computeapi.IsolatedDeviceConfig{ + DevType: devices[i].DevType, + Model: devices[i].Model, + DevicePath: devices[i].DevicePath, + } + data.IsolatedDevices = append(data.IsolatedDevices, isolatedDevice) + } + } + + // port mappings + // var portRange *computeapi.GuestPortMappingPortRange + portMappings := computeapi.GuestPortMappings{} + if sku.PortMappings != nil && !sku.PortMappings.IsZero() { + // hostTcpPortRange := computeapi.GuestPortMappingPortRange{ + // Start: options.Options.HostTcpPortStart, + // End: options.Options.HostTcpPortEnd, + // } + // hostUdpPortRange := computeapi.GuestPortMappingPortRange{ + // Start: options.Options.HostUdpPortStart, + // End: options.Options.HostUdpPortEnd, + // } + for _, portInfo := range *sku.PortMappings { + remoteIps := portInfo.RemoteIps + if len(remoteIps) == 0 { + remoteIps = nil + } + // if portInfo.Protocol == "tcp" { + // portRange = &hostTcpPortRange + // } else { + // portRange = &hostUdpPortRange + // } + portMappings = append(portMappings, &computeapi.GuestPortMapping{ + Port: portInfo.ContainerPort, + Protocol: computeapi.GuestPortMappingProtocol(portInfo.Protocol), + RemoteIps: remoteIps, + // HostPortRange: portRange, + Rule: &computeapi.GuestPortMappingRule{ + FirstPortOffset: portInfo.FirstPortOffset, + }, + Envs: portInfo.Envs, + }) + } + } + bandwidth := dify.BandwidthMb + if bandwidth == 0 { + bandwidth = sku.BandwidthMb + } + data.Networks = []*computeapi.NetworkConfig{ + { + NetType: computeapi.NETWORK_TYPE_HOSTLOCAL, + BwLimit: bandwidth, + PortMappings: portMappings, + }, + } + + data.Count = 1 + data.PreferHost = input.PreferHost + + ctrs := dify.GetDifyContainers() + + data.Pod = &computeapi.PodCreateInput{ + HostIPC: true, + Containers: ctrs, + } + + data.ProjectId = input.ProjectId + if len(data.ProjectId) == 0 { + data.ProjectId = userCred.GetProjectId() + data.TenantId = userCred.GetTenantId() + } + + return &data, nil +} diff --git a/pkg/llm/models/doc.go b/pkg/llm/models/doc.go new file mode 100644 index 0000000000..82c6e512b4 --- /dev/null +++ b/pkg/llm/models/doc.go @@ -0,0 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models // import "yunion.io/x/onecloud/pkg/llm/models" diff --git a/pkg/llm/models/image.go b/pkg/llm/models/image.go new file mode 100644 index 0000000000..5920414f95 --- /dev/null +++ b/pkg/llm/models/image.go @@ -0,0 +1,153 @@ +package models + +import ( + "context" + "fmt" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + "yunion.io/x/sqlchemy" + + identityapi "yunion.io/x/onecloud/pkg/apis/identity" + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/llm/options" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/mcclient/modules/identity" +) + +func init() { + GetLLMImageManager() +} + +var llmImageManager *SLLMImageManager + +func GetLLMImageManager() *SLLMImageManager { + if llmImageManager != nil { + return llmImageManager + } + llmImageManager = &SLLMImageManager{ + SSharableVirtualResourceBaseManager: db.NewSharableVirtualResourceBaseManager( + SLLMImage{}, + "llm_images_tbl", + "llm_image", + "llm_images", + ), + } + llmImageManager.SetVirtualObject(llmImageManager) + return llmImageManager +} + +type SLLMImageManager struct { + db.SSharableVirtualResourceBaseManager +} + +type SLLMImage struct { + db.SSharableVirtualResourceBase + + ImageName string `width:"128" charset:"utf8" nullable:"false" list:"user" create:"admin_optional" update:"user"` + ImageLabel string `width:"64" charset:"utf8" nullable:"false" list:"user" create:"admin_optional" update:"user"` + + CredentialId string `width:"128" charset:"utf8" nullable:"true" list:"user" create:"admin_optional" update:"user"` +} + +func fetchImageCredential(ctx context.Context, userCred mcclient.TokenCredential, cid string) (*identityapi.CredentialDetails, error) { + s := auth.GetSession(ctx, userCred, options.Options.Region) + credJson, err := identity.Credentials.Get(s, cid, nil) + if err != nil { + return nil, errors.Wrap(err, "Credentials.Get") + } + details := identityapi.CredentialDetails{} + err = credJson.Unmarshal(&details) + if err != nil { + return nil, errors.Wrap(err, "Unmarshal") + } + return &details, nil +} + +func (man *SLLMImageManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input *api.LLMImageCreateInput) (*api.LLMImageCreateInput, error) { + var err error + input.SharableVirtualResourceCreateInput, err = man.SSharableVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.SharableVirtualResourceCreateInput) + if nil != err { + return input, errors.Wrap(err, "validate SharableVirtualResourceCreateInput") + } + + if len(input.CredentialId) > 0 { + cred, err := fetchImageCredential(ctx, userCred, input.CredentialId) + if err != nil { + return input, errors.Wrap(err, "fetchImageCredential") + } + input.CredentialId = cred.Id + } + + input.Status = api.STATUS_READY + return input, nil +} + +func (man *SLLMImageManager) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input *api.LLMImageUpdateInput) (*api.LLMImageUpdateInput, error) { + var err error + input.SharableVirtualResourceCreateInput, err = man.SSharableVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.SharableVirtualResourceCreateInput) + if nil != err { + return input, errors.Wrap(err, "validate SharableVirtualResourceCreateInput") + } + + if nil != input.CredentialId && len(*input.CredentialId) > 0 { + cred, err := fetchImageCredential(ctx, userCred, *input.CredentialId) + if err != nil { + return input, errors.Wrap(err, "fetchImageCredential") + } + input.CredentialId = &cred.Id + } + return input, nil +} + +func (man *SLLMImageManager) ListItemFilter( + ctx context.Context, + q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, + input api.LLMImageListInput, +) (*sqlchemy.SQuery, error) { + q, err := man.SSharableVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, input.SharableVirtualResourceListInput) + if err != nil { + return nil, errors.Wrapf(err, "SSharableBaseResourceManager.ListItemFilter") + } + if input.IsPublic != nil { + if *input.IsPublic { + q = q.IsTrue("is_public") + } else { + q = q.IsFalse("is_public") + } + } + if len(input.ImageLabel) > 0 { + q = q.Equals("image_label", input.ImageLabel) + } + if len(input.ImageName) > 0 { + q = q.Equals("image_name", input.ImageName) + } + return q, nil +} + +func (image *SLLMImage) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error { + for _, field := range []string{"llm_image_id"} { + count, err := GetLLMManager().Query().Equals(field, image.Id).CountWithError() + if err != nil { + return errors.Wrap(err, "fetch llms") + } + if count > 0 { + return errors.Wrapf(errors.ErrNotSupported, "This image is currently in use by %s in llms", field) + } + count, err = GetLLMModelManager().Query().Equals("llm_image_id", image.Id).CountWithError() + if err != nil { + return errors.Wrap(err, "fetch llm models") + } + if count > 0 { + return errors.Wrapf(errors.ErrNotSupported, "This image is currently in use by %s in llm models", field) + } + } + return nil +} + +func (image *SLLMImage) ToContainerImage() string { + return fmt.Sprintf("%s:%s", image.ImageName, image.ImageLabel) +} diff --git a/pkg/llm/models/initdb.go b/pkg/llm/models/initdb.go new file mode 100644 index 0000000000..c19790c867 --- /dev/null +++ b/pkg/llm/models/initdb.go @@ -0,0 +1,25 @@ +package models + +import ( + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" +) + +func InitDB() error { + for _, manager := range []db.IModelManager{ + db.Metadata, + + // DifyManager, + // LLMManager, + } { + err := manager.InitializeData() + if err != nil { + log.Errorf("Manager %s initializeData fail %s", manager.Keyword(), err) + return err + } else { + log.Infof("Manager %s initializeData PASS!", manager.Keyword()) + } + } + return nil +} diff --git a/pkg/llm/models/llm.go b/pkg/llm/models/llm.go new file mode 100644 index 0000000000..d2307bdf01 --- /dev/null +++ b/pkg/llm/models/llm.go @@ -0,0 +1,321 @@ +package models + +import ( + "context" + "database/sql" + "strings" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/utils" + "yunion.io/x/sqlchemy" + + commonapi "yunion.io/x/onecloud/pkg/apis" + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/mcclient/modules/compute" +) + +var llmManager *SLLMManager + +func init() { + GetLLMManager() +} + +func GetLLMManager() *SLLMManager { + if llmManager != nil { + return llmManager + } + llmManager = &SLLMManager{ + SLLMBaseManager: NewSLLMBaseManager( + SLLM{}, + "llms_tbl", + "llm", + "llms", + ), + } + llmManager.SetVirtualObject(llmManager) + return llmManager +} + +type SLLMManager struct { + SLLMBaseManager +} + +type SLLM struct { + SLLMBase + + LLMModelId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"` + LLMImageId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"` +} + +func (man *SLLMManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input *api.LLMCreateInput) (*api.LLMCreateInput, error) { + var err error + input.LLMBaseCreateInput, err = man.SLLMBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.LLMBaseCreateInput) + if err != nil { + return input, errors.Wrap(err, "validate LLMBaseCreateInput") + } + model, err := GetLLMModelManager().FetchByIdOrName(ctx, userCred, input.LLMModelId) + if err != nil { + return input, errors.Wrap(err, "fetch LLMModel") + } + lModel := model.(*SLLMModel) + input.LLMModelId = lModel.Id + input.LLMImageId = lModel.LLMImageId + + return input, nil +} + +func (man *SLLMManager) BatchCreateValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input api.LLMCreateInput) (*jsonutils.JSONDict, error) { + data, err := man.ValidateCreateData(ctx, userCred, ownerId, query, &input) + if err != nil { + return nil, err + } + return data.JSON(data), nil +} + +func (man *SLLMManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, input api.LLMListInput) (*sqlchemy.SQuery, error) { + q, err := man.SLLMBaseManager.ListItemFilter(ctx, q, userCred, input.LLMBaseListInput) + if err != nil { + return q, errors.Wrap(err, "VirtualResourceBaseManager.ListItemFilter") + } + + if len(input.LLMModel) > 0 { + modelObj, err := GetLLMModelManager().FetchByIdOrName(ctx, userCred, input.LLMModel) + if err != nil { + if errors.Cause(err) == sql.ErrNoRows { + return nil, httperrors.NewResourceNotFoundError2(GetLLMModelManager().KeywordPlural(), input.LLMModel) + } else { + return nil, errors.Wrap(err, "LLMModelManager.FetchByIdOrName") + } + } + q = q.Equals("llm_model_id", modelObj.GetId()) + } + if len(input.LLMImage) > 0 { + imgObj, err := GetLLMImageManager().FetchByIdOrName(ctx, userCred, input.LLMImage) + if err != nil { + if errors.Cause(err) == sql.ErrNoRows { + return nil, httperrors.NewResourceNotFoundError2(GetLLMImageManager().KeywordPlural(), input.LLMImage) + } else { + return nil, errors.Wrap(err, "LLMImageManager.FetchByIdOrName") + } + } + q = q.Equals("llm_image_id", imgObj.GetId()) + } + + // if input.Unused != nil { + // instanceQ := GetDesktopInstanceManager().Query().SubQuery() + // if *input.Unused { + // q = q.NotEquals("id", instanceQ.Query(instanceQ.Field("desktop_id")).SubQuery()) + // } else { + // q = q.Join(instanceQ, sqlchemy.Equals(q.Field("id"), instanceQ.Field("desktop_id"))) + // } + // } + + return q, nil +} + +func (lm *SLLMManager) OnCreateComplete(ctx context.Context, items []db.IModel, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data []jsonutils.JSONObject) { + parentTaskId, _ := data[0].GetString("parent_task_id") + err := runBatchCreateTask(ctx, items, userCred, data, "LLMBatchCreateTask", parentTaskId) + if err != nil { + for i := range items { + llm := items[i].(*SLLM) + llm.SetStatus(ctx, userCred, api.LLM_STATUS_CREATE_FAIL, err.Error()) + } + } +} + +func runBatchCreateTask( + ctx context.Context, + items []db.IModel, + userCred mcclient.TokenCredential, + data []jsonutils.JSONObject, + taskName string, + parentTaskId string, +) error { + taskItems := make([]db.IStandaloneModel, len(items)) + for i, t := range items { + taskItems[i] = t.(db.IStandaloneModel) + } + params := jsonutils.NewDict() + params.Set("data", jsonutils.NewArray(data...)) + + task, err := taskman.TaskManager.NewParallelTask(ctx, taskName, taskItems, userCred, params, parentTaskId, "") + if err != nil { + return errors.Wrapf(err, "NewParallelTask %s", taskName) + } + + return task.ScheduleRun(nil) +} + +func (llm *SLLM) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error { + return llm.StartDeleteTask(ctx, userCred, "") +} + +func (llm *SLLM) GetLLMModel(modelId string) (*SLLMModel, error) { + if len(modelId) == 0 { + modelId = llm.LLMModelId + } + model, err := GetLLMModelManager().FetchById(modelId) + if err != nil { + return nil, errors.Wrap(err, "fetch LLMModel") + } + return model.(*SLLMModel), nil +} + +func (llm *SLLM) GetLargeLanguageModelName() (modelName string, modelTag string, err error) { + model, err := llm.GetLLMModel("") + if err != nil { + return "", "", err + } + name := model.LLMModelName + parts := strings.Split(name, ":") + modelName = parts[0] + modelTag = "latest" + if len(parts) > 1 { + modelTag = parts[1] + } + return +} + +func (llm *SLLM) GetLLMImage() (*SLLMImage, error) { + return llm.getImage(llm.LLMImageId) +} + +func (llm *SLLM) GetLLMContainer() (*SLLMContainer, error) { + return GetLLMContainerManager().FetchByLLMId(llm.Id) +} + +func (llm *SLLM) GetLLMContainerDriver() ILLMContainerDriver { + model, _ := llm.GetLLMModel(llm.LLMModelId) + return model.GetLLMContainerDriver() +} + +func (llm *SLLM) StartCreateTask(ctx context.Context, userCred mcclient.TokenCredential, input api.LLMCreateInput, parentTaskId string) error { + llm.SetStatus(ctx, userCred, commonapi.STATUS_CREATING, "") + params := jsonutils.Marshal(input).(*jsonutils.JSONDict) + var err = func() error { + task, err := taskman.TaskManager.NewTask(ctx, "LLMCreateTask", llm, userCred, params, parentTaskId, "", nil) + if err != nil { + return errors.Wrapf(err, "NewTask") + } + return task.ScheduleRun(params) + }() + if err != nil { + llm.SetStatus(ctx, userCred, api.LLM_STATUS_CREATE_FAIL, err.Error()) + return err + } + return nil +} + +func (llm *SLLM) StartPullModelTask(ctx context.Context, userCred mcclient.TokenCredential, input *jsonutils.JSONDict, parentTaskId string) error { + task, err := taskman.TaskManager.NewTask(ctx, "LLMPullModelTask", llm, userCred, input, parentTaskId, "", nil) + if err != nil { + return errors.Wrapf(err, "NewTask") + } + return task.ScheduleRun(nil) +} + +func (llm *SLLM) StartDeleteTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error { + llm.SetStatus(ctx, userCred, api.LLM_STATUS_START_DELETE, "StartDeleteTask") + task, err := taskman.TaskManager.NewTask(ctx, "LLMDeleteTask", llm, userCred, nil, parentTaskId, "", nil) + if err != nil { + return err + } + return task.ScheduleRun(nil) +} + +func (llm *SLLM) ServerCreate(ctx context.Context, userCred mcclient.TokenCredential, input *api.LLMCreateInput) (string, error) { + model, err := llm.GetLLMModel(llm.LLMModelId) + if nil != err { + return "", errors.Wrap(err, "GetLLMModel") + } + llmImage, err := llm.GetLLMImage() + if nil != err { + return "", errors.Wrap(err, "GetLLMImage") + } + + data, err := GetLLMPodCreateInput(ctx, userCred, input, llm, model, llmImage, "") + if nil != err { + return "", errors.Wrap(err, "GetPodCreateInput") + } + log.Infoln("PodCreateInput Data: ", jsonutils.Marshal(data).String()) + + s := auth.GetSession(ctx, userCred, "") + resp, err := compute.Servers.Create(s, jsonutils.Marshal(data)) + if nil != err { + return "", errors.Wrap(err, "Servers.Create") + } + id, err := resp.GetString("id") + if nil != err { + return "", errors.Wrap(err, "resp.GetString") + } + + return id, nil +} + +func (llm *SLLM) PerformStart(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + // can't start while it's already running + if utils.IsInStringArray(llm.Status, computeapi.VM_RUNNING_STATUS) { + return nil, errors.Wrapf(errors.ErrInvalidStatus, "llm id: %s status: %s", llm.Id, llm.Status) + } + + if err := llm.StartStartTask(ctx, userCred, ""); err != nil { + return nil, errors.Wrap(err, "StartStartTask") + } + return jsonutils.Marshal(nil), nil +} + +func (llm *SLLM) StartStartTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error { + task, err := taskman.TaskManager.NewTask(ctx, "LLMStartTask", llm, userCred, nil, parentTaskId, "", nil) + if err != nil { + return errors.Wrap(err, "NewTask") + } + return task.ScheduleRun(nil) +} + +func (llm *SLLM) PerformStop(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + if llm.Status == computeapi.VM_READY { + return nil, errors.Wrapf(errors.ErrInvalidStatus, "llm id: %s status: %s", llm.Id, llm.Status) + } + llm.SetStatus(ctx, userCred, computeapi.VM_START_STOP, "perform stop") + err := llm.StartLLMStopTask(ctx, userCred, "") + if err != nil { + return nil, errors.Wrap(err, "StartStopTask") + } + return nil, nil +} + +func (llm *SLLM) StartLLMStopTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error { + task, err := taskman.TaskManager.NewTask(ctx, "LLMStopTask", llm, userCred, nil, parentTaskId, "", nil) + if err != nil { + return errors.Wrap(err, "NewTask") + } + err = task.ScheduleRun(nil) + if err != nil { + return errors.Wrap(err, "ScheduleRun") + } + return nil +} + +// func (llm *SLLM) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error { +// instanceId, isBound, err := llm.IsBoundToInstance() +// if err != nil { +// return errors.Wrap(err, "IsBoundToInstance") +// } +// if isBound { +// return httperrors.NewBadRequestError("llm is bound to instance %s", instanceId) +// } +// return nil +// } + +// func (llm *SLLM) WaitContainerStatus(ctx context.Context, userCred mcclient.TokenCredential, targetStatus []string, timeoutSecs int) (*computeapi.SContainer, error) { +// return nil, nil +// } diff --git a/pkg/llm/models/llm_base.go b/pkg/llm/models/llm_base.go new file mode 100644 index 0000000000..23b76d4b8e --- /dev/null +++ b/pkg/llm/models/llm_base.go @@ -0,0 +1,307 @@ +package models + +import ( + "context" + "database/sql" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + "yunion.io/x/sqlchemy" + + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/llm/options" + cloudutil "yunion.io/x/onecloud/pkg/llm/utils" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/mcclient/modules/compute" + computeoptions "yunion.io/x/onecloud/pkg/mcclient/options/compute" + "yunion.io/x/onecloud/pkg/util/stringutils2" +) + +func NewSLLMBaseManager(dt interface{}, tableName string, keyword string, keywordPlural string) SLLMBaseManager { + return SLLMBaseManager{ + SVirtualResourceBaseManager: db.NewVirtualResourceBaseManager( + dt, + tableName, + keyword, + keywordPlural, + ), + } +} + +type SLLMBaseManager struct { + db.SVirtualResourceBaseManager + db.SEnabledResourceBaseManager +} + +type SLLMBase struct { + db.SVirtualResourceBase + db.SEnabledResourceBase + + SvrId string `width:"128" charset:"ascii" nullable:"true" list:"user"` + LLMIp string `width:"20" charset:"ascii" nullable:"true" list:"user"` + // Hypervisor string `width:"128" charset:"ascii" nullable:"true" list:"user"` + Priority int `nullable:"false" default:"100" list:"user"` + BandwidthMb int `nullable:"true" list:"user" create:"admin_optional"` + + LastAppProbe time.Time `nullable:"true" list:"user" create:"admin_optional"` + + // 是否请求同步更新镜像 + SyncImageRequest bool `default:"false" nullable:"false" list:"user" update:"user"` + + VolumeUsedMb int `nullable:"true" list:"user"` + VolumeUsedAt time.Time `nullable:"true" list:"user"` + + // 秒装应用配额（可安装的总容量限制） + // InstantAppQuotaGb int `list:"user" update:"user" create:"optional" default:"0" nullable:"false"` + + DebugMode bool `default:"false" nullable:"false" list:"user" update:"user"` + RootfsUnlimit bool `default:"false" nullable:"false" list:"user" update:"user"` +} + +func (man *SLLMBaseManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input api.LLMBaseCreateInput) (api.LLMBaseCreateInput, error) { + var err error + input.VirtualResourceCreateInput, err = man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.VirtualResourceCreateInput) + if err != nil { + return input, errors.Wrap(err, "validate VirtualResourceCreateInput") + } + + if len(input.PreferHost) > 0 { + s := auth.GetSession(ctx, userCred, "") + hostJson, err := compute.Hosts.Get(s, input.PreferHost, nil) + if err != nil { + return input, errors.Wrap(err, "get host") + } + hostDetails := computeapi.HostDetails{} + if err := hostJson.Unmarshal(&hostDetails); err != nil { + return input, errors.Wrap(err, "unmarshal hostDetails") + } + if hostDetails.Enabled == nil || !*hostDetails.Enabled { + return input, errors.Wrap(errors.ErrInvalidStatus, "not enabled") + } + if hostDetails.HostStatus != computeapi.HOST_ONLINE { + return input, errors.Wrap(errors.ErrInvalidStatus, "not online") + } + if hostDetails.HostType != computeapi.HOST_TYPE_CONTAINER { + return input, errors.Wrapf(httperrors.ErrNotAcceptable, "host_type %s not supported", hostDetails.HostType) + } + input.PreferHost = hostDetails.Id + } + + return input, nil +} + +func GetServerIdsByHost(ctx context.Context, userCred mcclient.TokenCredential, hostId string) ([]string, error) { + s := auth.GetSession(ctx, userCred, options.Options.Region) + params := computeoptions.ServerListOptions{} + params.Scope = "maxallowed" + params.Host = hostId + params.Field = []string{"id"} + limit := 1024 + params.Limit = &limit + offset := 0 + total := -1 + idList := stringutils2.NewSortedStrings(nil) + for total < 0 || offset < total { + params.Offset = &offset + results, err := compute.Servers.List(s, jsonutils.Marshal(params)) + if err != nil { + return nil, errors.Wrap(err, "Servers.List") + } + total = results.Total + for i := range results.Data { + idStr, _ := results.Data[i].GetString("id") + if len(idStr) > 0 { + idList = idList.Append(idStr) + } + } + offset += len(results.Data) + } + return idList, nil +} + +func (man *SLLMBaseManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, input api.LLMBaseListInput) (*sqlchemy.SQuery, error) { + q, err := man.SVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, input.VirtualResourceListInput) + if err != nil { + return q, errors.Wrap(err, "VirtualResourceBaseManager.ListItemFilter") + } + q, err = man.SEnabledResourceBaseManager.ListItemFilter(ctx, q, userCred, input.EnabledResourceBaseListInput) + if err != nil { + return q, errors.Wrap(err, "SEnabledResourceBaseManager.ListItemFilter") + } + + if len(input.Host) > 0 { + serverIds, err := GetServerIdsByHost(ctx, userCred, input.Host) + if err != nil { + return nil, errors.Wrap(err, "GetServerIdsByHost") + } + q = q.In("svr_id", serverIds) + } + if len(input.Status) > 0 { + s := auth.GetSession(ctx, userCred, options.Options.Region) + params := computeoptions.ServerListOptions{} + params.Scope = "maxallowed" + params.Status = input.Status + params.Field = []string{"guest_id"} + limit := 1024 + params.Limit = &limit + offset := 0 + total := -1 + idList := stringutils2.NewSortedStrings(nil) + for total < 0 || offset < total { + params.Offset = &offset + results, err := compute.Containers.List(s, jsonutils.Marshal(params)) + if err != nil { + return nil, errors.Wrap(err, "Containers.List") + } + total = results.Total + for i := range results.Data { + idStr, _ := results.Data[i].GetString("guest_id") + if len(idStr) > 0 { + idList = idList.Append(idStr) + } + } + offset += len(results.Data) + } + q = q.In("svr_id", idList) + } + + if input.NoVolume != nil { + volumeQ := GetVolumeManager().Query("llm_id").SubQuery() + q = q.LeftJoin(volumeQ, sqlchemy.Equals(q.Field("id"), volumeQ.Field("llm_id"))) + if *input.NoVolume { + q = q.Filter(sqlchemy.IsNull(volumeQ.Field("llm_id"))) + } else { + q = q.Filter(sqlchemy.IsNotNull(volumeQ.Field("llm_id"))) + } + } + if len(input.VolumeId) > 0 { + volumeObj, err := GetVolumeManager().FetchByIdOrName(ctx, userCred, input.VolumeId) + if err != nil { + return nil, errors.Wrap(err, "VolumeManager.FetchByIdOrName") + } + vq := GetVolumeManager().Query().SubQuery() + q = q.Join(vq, sqlchemy.Equals(q.Field("id"), vq.Field("llm_id"))) + q = q.Filter(sqlchemy.Equals(vq.Field("id"), volumeObj.GetId())) + } + + accessQ := GetAccessInfoManager().Query().SubQuery() + if input.ListenPort > 0 { + q = q.Join(accessQ, sqlchemy.Equals(q.Field("id"), accessQ.Field("llm_id"))) + q = q.Filter(sqlchemy.Equals(accessQ.Field("listen_port"), input.ListenPort)) + } + + if len(input.PublicIp) > 0 { + s := auth.GetSession(ctx, userCred, "") + hostInput := computeapi.HostListInput{ + PublicIp: []string{input.PublicIp}, + } + hostInput.Field = []string{"id"} + hosts, err := compute.Hosts.List(s, jsonutils.Marshal(hostInput)) + if err != nil { + return nil, errors.Wrap(err, "Hosts.List") + } + if len(hosts.Data) == 0 { + return nil, httperrors.NewNotFoundError("Not found host by public_ip %s", input.PublicIp) + } + hostIds := []string{} + for i := range hosts.Data { + idStr, _ := hosts.Data[i].GetString("id") + if len(idStr) > 0 { + hostIds = append(hostIds, idStr) + } + } + if len(hostIds) > 0 { + serverIds, err := GetServerIdsByHost(ctx, userCred, hostIds[0]) + if err != nil { + return nil, errors.Wrap(err, "GetServerIdsByHost") + } + q = q.In("svr_id", serverIds) + } + } + // if input.Unused != nil { + // instanceQ := GetDesktopInstanceManager().Query().SubQuery() + // if *input.Unused { + // q = q.NotEquals("id", instanceQ.Query(instanceQ.Field("desktop_id")).SubQuery()) + // } else { + // q = q.Join(instanceQ, sqlchemy.Equals(q.Field("id"), instanceQ.Field("desktop_id"))) + // } + // } + + return q, nil +} + +func (llm *SLLMBase) GetServer(ctx context.Context) (*computeapi.ServerDetails, error) { + return cloudutil.GetServer(ctx, llm.SvrId) +} + +func (llm *SLLMBase) GetVolume() (*SVolume, error) { + volume := &SVolume{} + err := GetVolumeManager().Query().Equals("llm_id", llm.Id).First(volume) + if err != nil { + if errors.Cause(err) == sql.ErrNoRows { + return nil, errors.Wrap(errors.ErrNotFound, "query volume") + } + return nil, errors.Wrap(err, "FetchVolume") + } + volume.SetModelManager(GetVolumeManager(), volume) + return volume, nil +} + +// 取消自动删除 +func (llm *SLLMBase) Delete(ctx context.Context, userCred mcclient.TokenCredential) error { + return nil +} + +func (llm *SLLMBase) RealDelete(ctx context.Context, userCred mcclient.TokenCredential) error { + return llm.SVirtualResourceBase.Delete(ctx, userCred) +} + +func (llm *SLLMBase) ServerDelete(ctx context.Context, userCred mcclient.TokenCredential) error { + if len(llm.SvrId) == 0 { + return nil + } + s := auth.GetSession(ctx, userCred, "") + server, err := llm.GetServer(ctx) + if err != nil { + if errors.Cause(err) == errors.ErrNotFound { + return nil + } else { + return errors.Wrap(err, "GetServer") + } + } + if server.DisableDelete != nil && *server.DisableDelete { + // update to allow delete + _, err = compute.Servers.Update(s, llm.SvrId, jsonutils.Marshal(map[string]interface{}{"disable_delete": false})) + if err != nil { + return errors.Wrap(err, "update server to delete") + } + } + _, err = compute.Servers.DeleteWithParam(s, llm.SvrId, jsonutils.Marshal(map[string]interface{}{ + "override_pending_delete": true, + }), nil) + if err != nil { + return errors.Wrap(err, "delete server err:") + } + return nil +} + +func (llm *SLLMBase) WaitDelete(ctx context.Context, userCred mcclient.TokenCredential, timeoutSecs int) error { + return cloudutil.WaitDelete[computeapi.ServerDetails](ctx, &compute.Servers, llm.SvrId, timeoutSecs) +} + +func (llm *SLLMBase) getImage(imageId string) (*SLLMImage, error) { + image, err := GetLLMImageManager().FetchById(imageId) + if err != nil { + return nil, errors.Wrap(err, "fetch LLMImage") + } + return image.(*SLLMImage), nil +} + +func (llm *SLLMBase) WaitServerStatus(ctx context.Context, userCred mcclient.TokenCredential, targetStatus []string, timeoutSecs int) (*computeapi.ServerDetails, error) { + return cloudutil.WaitServerStatus(ctx, llm.SvrId, targetStatus, timeoutSecs) +} diff --git a/pkg/llm/models/llm_container.go b/pkg/llm/models/llm_container.go new file mode 100644 index 0000000000..9b97dbaae2 --- /dev/null +++ b/pkg/llm/models/llm_container.go @@ -0,0 +1,91 @@ +package models + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + commonapi "yunion.io/x/onecloud/pkg/apis" + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/mcclient" +) + +var ( + llmContainerManager *SLLMContainerManager +) + +func GetLLMContainerManager() *SLLMContainerManager { + if llmContainerManager != nil { + return llmContainerManager + } + m := NewLLMContainerManager(SLLMContainer{}, "llm_container", "llm_containers") + llmContainerManager = &m + llmContainerManager.SetVirtualObject(llmContainerManager) + return llmContainerManager +} + +func NewLLMContainerManager(dt interface{}, keyword string, keywordPlural string) SLLMContainerManager { + return SLLMContainerManager{ + SVirtualResourceBaseManager: db.NewVirtualResourceBaseManager(dt, "llm_containers_tbl", keyword, keywordPlural), + } +} + +type SLLMContainerManager struct { + db.SVirtualResourceBaseManager +} + +type SLLMContainer struct { + db.SVirtualResourceBase + + LLMId string `width:"128" charset:"ascii" nullable:"false" list:"user" primary:"true" create:"required"` + CmpId string `width:"128" charset:"ascii" nullable:"true" list:"user" create:"required"` + Type string `width:"16" charset:"ascii" list:"user" primary:"true" create:"required"` + RunningAppId string `width:"128" charset:"ascii" nullable:"true" list:"user"` +} + +func (m *SLLMContainerManager) CreateOnLLM( + ctx context.Context, + userCred mcclient.TokenCredential, + ownerId mcclient.IIdentityProvider, + llm *SLLM, cmpId string, svrName string, +) (*SLLMContainer, error) { + input := &api.LLMContainerCreateInput{ + LLMId: llm.Id, + Type: string(llm.GetLLMContainerDriver().GetType()), + CmpId: cmpId, + } + input.Name = svrName + obj, err := db.DoCreate(m, ctx, userCred, nil, jsonutils.Marshal(input), ownerId) + if err != nil { + return nil, errors.Wrap(err, "create llm container") + } + return obj.(*SLLMContainer), nil +} + +func (m *SLLMContainerManager) FetchByLLMId( + llmId string, +) (*SLLMContainer, error) { + lc := &SLLMContainer{} + if err := m.Query().Equals("llm_id", llmId).First(lc); err != nil { + return nil, errors.Wrapf(err, "query llm container by llm id %s", llmId) + } + lc.SetModelManager(m, lc) + return lc, nil +} + +func (lc *SLLMContainer) RealDelete(ctx context.Context, userCred mcclient.TokenCredential) error { + return lc.SVirtualResourceBase.Delete(ctx, userCred) +} + +func (lc *SLLMContainer) StartDeleteTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error { + task, err := taskman.TaskManager.NewTask(ctx, "LLMContainerDeleteTask", lc, userCred, nil, parentTaskId, "", nil) + if err != nil { + return err + } + lc.SetStatus(ctx, userCred, commonapi.STATUS_DELETING, "") + task.ScheduleRun(nil) + return nil +} diff --git a/pkg/llm/models/llm_container_driver.go b/pkg/llm/models/llm_container_driver.go new file mode 100644 index 0000000000..c1317f79e5 --- /dev/null +++ b/pkg/llm/models/llm_container_driver.go @@ -0,0 +1,90 @@ +package models + +import ( + "context" + "sync" + + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" +) + +type drivers struct { + drivers *sync.Map +} + +func newDrivers() *drivers { + return &drivers{ + drivers: &sync.Map{}, + } +} + +func (d *drivers) GetWithError(typ string) (interface{}, error) { + drv, ok := d.drivers.Load(typ) + if !ok { + return drv, httperrors.NewNotFoundError("app container driver %s not found", typ) + } + return drv, nil +} + +func (d *drivers) Get(typ string) interface{} { + drv, err := d.GetWithError(typ) + if err != nil { + panic(err.Error()) + } + return drv +} + +func (d *drivers) Register(typ string, drv interface{}) { + d.drivers.Store(typ, drv) +} + +func registerDriver[K ~string, D any](drvs *drivers, typ K, drv D) { + drvs.Register(string(typ), drv) +} + +func getDriver[K ~string, D any](drvs *drivers, typ K) D { + return drvs.Get(string(typ)).(D) +} + +func getDriverWithError[K ~string, D any](drvs *drivers, typ K) (D, error) { + drv, err := drvs.GetWithError(string(typ)) + if err != nil { + return drv.(D), err + } + return drv.(D), nil +} + +type ILLMContainerPullModel interface { + // PullModelByInstall(ctx context.Context, userCred mcclient.TokenCredential, llm *SLLM, modelName string, modelTag string) error + // PullModelByGgufFile(ctx context.Context, userCred mcclient.TokenCredential, llm *SLLM, ggufFileUrl string, model string) error + // DownloadGgufFile(ctx context.Context, userCred mcclient.TokenCredential, llm *SLLM, ggufFileUrl string, ggufFilePath string) error + // InstallGgufModel(ctx context.Context, userCred mcclient.TokenCredential, llm *SLLM, ggufFilePath string) error + GetManifests(ctx context.Context, userCred mcclient.TokenCredential, llm *SLLM, taskId string) error + AccessBlobsCache(ctx context.Context, userCred mcclient.TokenCredential, llm *SLLM, taskId string) error + CopyBlobs(ctx context.Context, userCred mcclient.TokenCredential, llm *SLLM) error +} + +type ILLMContainerDriver interface { + GetType() llm.LLMContainerType + GetContainerSpec(ctx context.Context, llm *SLLM, image *SLLMImage, sku *SLLMModel, props []string, devices []computeapi.SIsolatedDevice, diskId string) *computeapi.PodContainerCreateInput + + // ILLMContainerPullModel +} + +var ( + llmContainerDrivers = newDrivers() +) + +func RegisterLLMContainerDriver(drv ILLMContainerDriver) { + registerDriver(llmContainerDrivers, drv.GetType(), drv) +} + +func GetLLMContainerDriver(typ llm.LLMContainerType) ILLMContainerDriver { + return getDriver[llm.LLMContainerType, ILLMContainerDriver](llmContainerDrivers, typ) +} + +func GetLLMContainerDriverWithError(typ llm.LLMContainerType) (ILLMContainerDriver, error) { + return getDriverWithError[llm.LLMContainerType, ILLMContainerDriver](llmContainerDrivers, typ) +} diff --git a/pkg/llm/models/llm_model.go b/pkg/llm/models/llm_model.go new file mode 100644 index 0000000000..6450bae80b --- /dev/null +++ b/pkg/llm/models/llm_model.go @@ -0,0 +1,199 @@ +package models + +import ( + "context" + "strings" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/sqlchemy" + + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/validators" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/stringutils2" +) + +func init() { + GetLLMModelManager() +} + +var llmModelManager *SLLMModelManager + +func GetLLMModelManager() *SLLMModelManager { + if llmModelManager != nil { + return llmModelManager + } + llmModelManager = &SLLMModelManager{ + SLLMModelBaseManager: NewSLLMModelBaseManager( + SLLMModel{}, + "llm_models_tbl", + "llm_model", + "llm_models", + ), + } + llmModelManager.SetVirtualObject(llmModelManager) + return llmModelManager +} + +type SLLMModelManager struct { + SLLMModelBaseManager +} + +type SLLMModel struct { + SLLMModelBase + // SMountedAppsResource + + LLMImageId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"` + LLMType string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"` + LLMModelName string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"` +} + +func (man *SLLMModelManager) ListItemFilter( + ctx context.Context, + q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, + input api.LLMModelListInput, +) (*sqlchemy.SQuery, error) { + var err error + q, err = man.SLLMModelBaseManager.ListItemFilter(ctx, q, userCred, input.SharableVirtualResourceListInput) + if err != nil { + return nil, errors.Wrapf(err, "SLLMModelBaseManager.ListItemFilter") + } + if len(input.LLMType) > 0 { + q = q.Equals("llm_type", input.LLMType) + } + // q, err = man.SMountedAppsResourceManager.ListItemFilter(ctx, q, userCred, input.MountedAppResourceListInput) + // if err != nil { + // return nil, errors.Wrap(err, "SMountedAppsResourceManager") + // } + return q, nil +} + +func (manager *SLLMModelManager) FetchCustomizeColumns( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + objs []interface{}, + fields stringutils2.SSortedStrings, + isList bool, +) []api.LLMModelDetails { + // skuIds := []string{} + imageIds := []string{} + // templateIds := []string{} + + skus := []SLLMModel{} + jsonutils.Update(&skus, objs) + virows := manager.SSharableVirtualResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) + for _, sku := range skus { + // skuIds = append(skuIds, sku.Id) + imageIds = append(imageIds, sku.LLMImageId) + // if sku.Volumes != nil && len(*sku.Volumes) > 0 && len((*sku.Volumes)[0].TemplateId) > 0 { + // templateIds = append(templateIds, (*sku.Volumes)[0].TemplateId) + // } + } + + // q := GetLLMManager().Query().In("llm_model_id", skuIds).GroupBy("llm_model_id") + // q = q.AppendField(q.Field("llm_model_id")) + // q = q.AppendField(sqlchemy.COUNT("llm_capacity")) + // details := []struct { + // LLMModelId string + // LLMCapacity int + // }{} + // q.All(&details) + res := make([]api.LLMModelDetails, len(objs)) + for i := range skus { + res[i].SharableVirtualResourceDetails = virows[i] + // for _, v := range details { + // if v.LLMModelId == sku.Id { + // res[i].LLMCapacity = v.LLMCapacity + // break + // } + // } + } + { + images := make(map[string]SLLMImage) + err := db.FetchModelObjectsByIds(GetLLMImageManager(), "id", imageIds, &images) + if err == nil { + for i, sku := range skus { + if image, ok := images[sku.LLMImageId]; ok { + res[i].Image = image.Name + res[i].ImageLabel = image.ImageLabel + res[i].ImageName = image.ImageName + } + } + } else { + log.Errorf("FetchModelObjectsByIds DesktopImageManager fail %s", err) + } + } + + // if len(templateIds) > 0 { + // templates, err := fetchTemplates(ctx, userCred, templateIds) + // if err == nil { + // for i, sku := range skus { + // if templ, ok := templates[(*sku.Volumes)[0].TemplateId]; ok { + // res[i].Template = templ.Name + // } + // } + // } else { + // log.Errorf("fail to retrive image info %s", err) + // } + // } + + return res +} + +func (man *SLLMModelManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input *api.LLMModelCreateInput) (*api.LLMModelCreateInput, error) { + var err error + input.LLMModelBaseCreateInput, err = man.SLLMModelBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.LLMModelBaseCreateInput) + if err != nil { + return nil, errors.Wrap(err, "SLLMModelBaseManager.ValidateCreateData") + } + if !api.IsLLMContainerType(input.LLMType) { + return input, errors.Wrap(httperrors.ErrInputParameter, "llm_type must be one of "+strings.Join(api.LLM_CONTAINER_TYPES.List(), ",")) + } + + _, err = validators.ValidateModel(ctx, userCred, GetLLMImageManager(), &input.LLMImageId) + if err != nil { + return input, errors.Wrapf(err, "validate image_id %s", input.LLMImageId) + } + + input.Status = api.STATUS_READY + return input, nil +} + +func (model *SLLMModel) GetLLMContainerDriver() ILLMContainerDriver { + return GetLLMContainerDriver(api.LLMContainerType(model.LLMType)) +} + +func (model *SLLMModel) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.LLMModelUpdateInput) (api.LLMModelUpdateInput, error) { + var err error + input.LLMModelBaseUpdateInput, err = model.SLLMModelBase.ValidateUpdateData(ctx, userCred, query, input.LLMModelBaseUpdateInput) + if err != nil { + return input, errors.Wrap(err, "validate LLMModelBaseUpdateInput") + } + + if input.LLMImageId != "" { + imgObj, err := validators.ValidateModel(ctx, userCred, GetLLMImageManager(), &input.LLMImageId) + if err != nil { + return input, errors.Wrapf(err, "validate image_id %s", input.LLMImageId) + } + input.LLMImageId = imgObj.GetId() + } + + return input, nil +} + +func (model *SLLMModel) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error { + count, err := GetLLMManager().Query().Equals("llm_model_id", model.Id).CountWithError() + if nil != err { + return errors.Wrap(err, "fetch llm") + } + if count > 0 { + return errors.Wrap(errors.ErrNotSupported, "This model is currently in use") + } + return nil +} diff --git a/pkg/llm/models/llm_pod.go b/pkg/llm/models/llm_pod.go new file mode 100644 index 0000000000..b0d3d9335c --- /dev/null +++ b/pkg/llm/models/llm_pod.go @@ -0,0 +1,172 @@ +package models + +import ( + "context" + "fmt" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/apis" + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/mcclient" +) + +const ( + POD_METADATA_POST_STOP_CLEANUP_CONFIG = "post_stop_cleanup_config" +) + +type PodPostStopCleanupConfig struct { + Dirs []string `json:"dirs"` +} + +func GetLLMPodCreateInput( + ctx context.Context, + userCred mcclient.TokenCredential, + input *api.LLMCreateInput, + llm *SLLM, + sku *SLLMModel, + llmImage *SLLMImage, + eip string, +) (*computeapi.ServerCreateInput, error) { + data := computeapi.ServerCreateInput{} + data.AutoStart = input.AutoStart + data.ServerConfigs = computeapi.NewServerConfigs() + data.Hypervisor = computeapi.HYPERVISOR_POD + + postStopCleanupConfgi := PodPostStopCleanupConfig{ + Dirs: []string{ + GetTmpHostPath(llm.GetName()), + }, + } + data.Metadata = map[string]string{ + POD_METADATA_POST_STOP_CLEANUP_CONFIG: jsonutils.Marshal(postStopCleanupConfgi).String(), + } + + data.VcpuCount = sku.Cpu + data.VmemSize = sku.Memory + 1 + data.Name = input.Name + + // disks + data.Disks = make([]*computeapi.DiskConfig, 0) + if sku.Volumes != nil && !sku.Volumes.IsZero() { + for idx, volume := range *sku.Volumes { + data.Disks = append(data.Disks, &computeapi.DiskConfig{ + DiskType: "data", + Format: "raw", + Fs: "ext4", + SizeMb: volume.SizeMB, + Index: idx, + }) + } + } + + // isolated devices + if sku.Devices != nil && !sku.Devices.IsZero() { + data.IsolatedDevices = make([]*computeapi.IsolatedDeviceConfig, 0) + devices := *sku.Devices + for i := 0; i < len(devices); i++ { + isolatedDevice := &computeapi.IsolatedDeviceConfig{ + DevType: devices[i].DevType, + Model: devices[i].Model, + DevicePath: devices[i].DevicePath, + } + data.IsolatedDevices = append(data.IsolatedDevices, isolatedDevice) + } + } + + // port mappings + // var portRange *computeapi.GuestPortMappingPortRange + portMappings := computeapi.GuestPortMappings{} + if sku.PortMappings != nil && !sku.PortMappings.IsZero() { + // hostTcpPortRange := computeapi.GuestPortMappingPortRange{ + // Start: options.Options.HostTcpPortStart, + // End: options.Options.HostTcpPortEnd, + // } + // hostUdpPortRange := computeapi.GuestPortMappingPortRange{ + // Start: options.Options.HostUdpPortStart, + // End: options.Options.HostUdpPortEnd, + // } + for _, portInfo := range *sku.PortMappings { + remoteIps := portInfo.RemoteIps + if len(remoteIps) == 0 { + remoteIps = nil + } + // if portInfo.Protocol == "tcp" { + // portRange = &hostTcpPortRange + // } else { + // portRange = &hostUdpPortRange + // } + portMappings = append(portMappings, &computeapi.GuestPortMapping{ + Port: portInfo.ContainerPort, + Protocol: computeapi.GuestPortMappingProtocol(portInfo.Protocol), + RemoteIps: remoteIps, + // HostPortRange: portRange, + Rule: &computeapi.GuestPortMappingRule{ + FirstPortOffset: portInfo.FirstPortOffset, + }, + Envs: portInfo.Envs, + }) + } + } + bandwidth := llm.BandwidthMb + if bandwidth == 0 { + bandwidth = sku.BandwidthMb + } + data.Networks = []*computeapi.NetworkConfig{ + { + NetType: computeapi.NETWORK_TYPE_HOSTLOCAL, + BwLimit: bandwidth, + PortMappings: portMappings, + }, + } + + data.Count = 1 + data.PreferHost = input.PreferHost + + // enableLxcfs := true + + lcd := llm.GetLLMContainerDriver() + llmContainer := lcd.GetContainerSpec(ctx, llm, llmImage, sku, nil, nil, "") + + data.Pod = &computeapi.PodCreateInput{ + HostIPC: true, + Containers: []*computeapi.PodContainerCreateInput{ + llmContainer, + }, + } + + data.ProjectId = input.ProjectId + if len(data.ProjectId) == 0 { + data.ProjectId = userCred.GetProjectId() + data.TenantId = userCred.GetTenantId() + } + + return &data, nil +} + +func NewHostDev(path string) *computeapi.ContainerDevice { + return &computeapi.ContainerDevice{ + Type: apis.CONTAINER_DEVICE_TYPE_HOST, + Host: &computeapi.ContainerHostDevice{ + HostPath: path, + ContainerPath: path, + Permissions: "rwm", + }, + } +} + +func NewEnv(key, val string) *apis.ContainerKeyValue { + return &apis.ContainerKeyValue{ + Key: key, + Value: val, + } +} + +func GetTmpHostPath(name string) string { + return fmt.Sprintf("/tmp/%s", name) +} + +func GetSvrLLMContainer(ctrs []*computeapi.PodContainerDesc) *computeapi.PodContainerDesc { + return ctrs[0] +} diff --git a/pkg/llm/models/model.go b/pkg/llm/models/model.go new file mode 100644 index 0000000000..dd5e984aa9 --- /dev/null +++ b/pkg/llm/models/model.go @@ -0,0 +1,115 @@ +package models + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + "yunion.io/x/sqlchemy" + + "yunion.io/x/onecloud/pkg/apis" + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" +) + +func NewSLLMModelBaseManager(dt interface{}, tableName string, keyword string, keywordPlural string) SLLMModelBaseManager { + return SLLMModelBaseManager{ + SSharableVirtualResourceBaseManager: db.NewSharableVirtualResourceBaseManager( + dt, + tableName, + keyword, + keywordPlural, + ), + } +} + +type SLLMModelBaseManager struct { + db.SSharableVirtualResourceBaseManager +} + +type SLLMModelBase struct { + db.SSharableVirtualResourceBase + + BandwidthMb int `nullable:"false" default:"0" create:"optional" list:"user" update:"user"` + Cpu int `nullable:"false" default:"1" create:"optional" list:"user" update:"user"` + Memory int `nullable:"false" default:"512" create:"optional" list:"user" update:"user"` + Volumes *api.Volumes `charset:"utf8" length:"medium" nullable:"true" list:"user" update:"user" create:"optional"` + PortMappings *api.PortMappings `charset:"utf8" length:"medium" nullable:"true" list:"user" update:"user" create:"optional"` + Devices *api.Devices `charset:"utf8" length:"medium" nullable:"true" list:"user" update:"user" create:"optional"` + Envs *api.Envs `charset:"utf8" nullable:"true" list:"user" update:"user" create:"optional"` + // Properties + Properties map[string]string `charset:"utf8" nullable:"true" list:"user" update:"user" create:"optional"` +} + +func (man *SLLMModelBaseManager) ListItemFilter( + ctx context.Context, + q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, + input apis.SharableVirtualResourceListInput, +) (*sqlchemy.SQuery, error) { + var err error + q, err = man.SSharableVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, input) + if err != nil { + return nil, errors.Wrapf(err, "SSharableBaseResourceManager.ListItemFilter") + } + return q, nil +} + +func (man *SLLMModelBaseManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input api.LLMModelBaseCreateInput) (api.LLMModelBaseCreateInput, error) { + var err error + input.SharableVirtualResourceCreateInput, err = man.SSharableVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.SharableVirtualResourceCreateInput) + if err != nil { + return input, errors.Wrap(err, "SSharableVirtualResourceBaseManager.ValidateCreateData") + } + if input.Cpu <= 0 { + return input, errors.Wrap(httperrors.ErrInputParameter, "cpu must > 0") + } + if input.Memory <= 0 { + return input, errors.Wrap(httperrors.ErrInputParameter, "mem must > 0") + } + if input.Volumes == nil { + return input, errors.Wrap(httperrors.ErrInputParameter, "volumes cannot be empty") + } + + input.Status = api.STATUS_READY + return input, nil +} + +func (modelBase *SLLMModelBase) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.LLMModelBaseUpdateInput) (api.LLMModelBaseUpdateInput, error) { + var err error + input.SharableVirtualResourceBaseUpdateInput, err = modelBase.SSharableVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, input.SharableVirtualResourceBaseUpdateInput) + if err != nil { + return input, errors.Wrap(err, "validate SharableVirtualResourceBaseUpdateInput") + } + + volumes := []api.Volume{} + if err := jsonutils.Marshal(modelBase.Volumes).Unmarshal(&volumes); err != nil { + return input, errors.Wrapf(err, "Unmarshal Volumes") + } + for i, volume := range volumes { + if input.DiskSizeMB != nil && *input.DiskSizeMB > 0 { + volume.SizeMB = *input.DiskSizeMB + } + // if input.TemplateId != nil { + // if len(*input.TemplateId) > 0 { + // s := auth.GetSession(ctx, userCred, "") + // imgObj, err := imagemodules.Images.Get(s, *input.TemplateId, nil) + // if err != nil { + // return input, errors.Wrapf(err, "validate template_id %s", *input.TemplateId) + // } + // volume.TemplateId, _ = imgObj.GetString("id") + // } else { + // volume.TemplateId = "" + // } + // } + if input.StorageType != nil && len(*input.StorageType) > 0 { + volume.StorageType = *input.StorageType + } + volumes[i] = volume + } + input.Volumes = (*api.Volumes)(&volumes) + + return input, nil +} diff --git a/pkg/llm/models/volume.go b/pkg/llm/models/volume.go new file mode 100644 index 0000000000..ca5d122e02 --- /dev/null +++ b/pkg/llm/models/volume.go @@ -0,0 +1,67 @@ +package models + +import ( + "context" + + commonapi "yunion.io/x/onecloud/pkg/apis" + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/mcclient" +) + +func init() { + GetVolumeManager() +} + +var volumeManager *SVolumeManager + +func GetVolumeManager() *SVolumeManager { + if volumeManager != nil { + return volumeManager + } + volumeManager = &SVolumeManager{ + SVirtualResourceBaseManager: db.NewVirtualResourceBaseManager( + SVolume{}, + "volumes_tbl", + "llm_volume", + "llm_volumes", + ), + } + volumeManager.SetVirtualObject(volumeManager) + return volumeManager +} + +type SVolumeManager struct { + db.SVirtualResourceBaseManager + // SMountedAppsResourceManager +} + +type SVolume struct { + db.SVirtualResourceBase + // SMountedAppsResource + + LLMId string `width:"128" charset:"ascii" nullable:"true" list:"user" create:"admin_optional" update:"user"` + // 存储类型 + StorageType string `width:"16" charset:"ascii" nullable:"true" list:"user" create:"admin_optional" update:"user"` + // 模板ID + TemplateId string `width:"128" charset:"ascii" nullable:"true" list:"user" create:"admin_optional" update:"user"` + // size in MB + SizeMB int `nullable:"false" default:"0" create:"optional" list:"user" update:"user"` + SvrId string `width:"128" charset:"ascii" nullable:"true" list:"user"` + Containers api.ContainerVolumeRelations `charset:"utf8" nullable:"true" list:"user" create:"optional"` +} + +func (volume *SVolume) StartDeleteTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error { + task, err := taskman.TaskManager.NewTask(ctx, "VolumeDeleteTask", volume, userCred, nil, parentTaskId, "", nil) + if err != nil { + return err + } + volume.SetStatus(ctx, userCred, commonapi.STATUS_DELETING, "") + task.ScheduleRun(nil) + return nil +} + +func (volume *SVolume) RealDelete(ctx context.Context, userCred mcclient.TokenCredential) error { + return volume.SVirtualResourceBase.Delete(ctx, userCred) +} diff --git a/pkg/llm/options/doc.go b/pkg/llm/options/doc.go new file mode 100644 index 0000000000..bbb8556e07 --- /dev/null +++ b/pkg/llm/options/doc.go @@ -0,0 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package options // import "yunion.io/x/onecloud/pkg/llm/options" diff --git a/pkg/llm/options/option.go b/pkg/llm/options/option.go new file mode 100644 index 0000000000..98a748dd74 --- /dev/null +++ b/pkg/llm/options/option.go @@ -0,0 +1,28 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package options + +import common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" + +type LLMOptions struct { + common_options.CommonOptions + common_options.DBOptions + + StartTaskWorkerCount int `help:"start task worker count" default:"128"` +} + +var ( + Options LLMOptions +) diff --git a/pkg/llm/service/doc.go b/pkg/llm/service/doc.go new file mode 100644 index 0000000000..baa5463b8c --- /dev/null +++ b/pkg/llm/service/doc.go @@ -0,0 +1 @@ +package service // import "yunion.io/x/onecloud/pkg/llm/service" diff --git a/pkg/llm/service/handler.go b/pkg/llm/service/handler.go new file mode 100644 index 0000000000..55ee3add22 --- /dev/null +++ b/pkg/llm/service/handler.go @@ -0,0 +1,47 @@ +package service + +import ( + "yunion.io/x/onecloud/pkg/appsrv" + "yunion.io/x/onecloud/pkg/appsrv/dispatcher" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/llm/models" +) + +func InitHandlers(app *appsrv.Application) { + db.InitAllManagers() + db.RegistUserCredCacheUpdater() + + taskman.AddTaskHandler("", app) + + for _, manager := range []db.IModelManager{ + taskman.TaskManager, + taskman.SubTaskManager, + taskman.TaskObjectManager, + taskman.ArchivedTaskManager, + + db.SharedResourceManager, + db.UserCacheManager, + db.TenantCacheManager, + } { + db.RegisterModelManager(manager) + } + + for _, manager := range []db.IModelManager{ + db.OpsLog, + db.Metadata, + + models.GetLLMImageManager(), + models.GetLLMModelManager(), + models.GetDifyModelManager(), + models.GetVolumeManager(), + models.GetAccessInfoManager(), + models.GetLLMContainerManager(), + models.GetLLMManager(), + models.GetDifyManager(), + } { + db.RegisterModelManager(manager) + handler := db.NewModelHandler(manager) + dispatcher.AddModelDispatcher("", app, handler) + } +} diff --git a/pkg/llm/service/service.go b/pkg/llm/service/service.go new file mode 100644 index 0000000000..9cdd8c7573 --- /dev/null +++ b/pkg/llm/service/service.go @@ -0,0 +1,47 @@ +package service + +import ( + "os" + + "yunion.io/x/log" + _ "yunion.io/x/sqlchemy/backends" + + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/cloudcommon" + app_common "yunion.io/x/onecloud/pkg/cloudcommon/app" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" + _ "yunion.io/x/onecloud/pkg/llm/drivers/llm_container" + "yunion.io/x/onecloud/pkg/llm/models" + "yunion.io/x/onecloud/pkg/llm/options" + _ "yunion.io/x/onecloud/pkg/llm/tasks" +) + +// StartService the main service starts +func StartService() { + opts := &options.Options + commonOpts := &opts.CommonOptions + dbOpts := &options.Options.DBOptions + baseOpts := &opts.BaseOptions + common_options.ParseOptions(opts, os.Args, "llm.conf", api.SERVICE_TYPE) + + app_common.InitAuth(commonOpts, func() { + log.Infof("Auth complete!!") + }) + + app := app_common.InitApp(&opts.BaseOptions, false) + + cloudcommon.InitDB(dbOpts) + InitHandlers(app) + + db.EnsureAppSyncDB(app, dbOpts, models.InitDB) + defer cloudcommon.CloseDB() + + // if !opts.IsSlaveNode { + // models.InitializeCronjobs(app.GetContext()) + // } + + app_common.ServeForeverWithCleanup(app, baseOpts, func() { + cloudcommon.CloseDB() + }) +} diff --git a/pkg/llm/tasks/dify/dify_batch_create_task.go b/pkg/llm/tasks/dify/dify_batch_create_task.go new file mode 100644 index 0000000000..e53da625ca --- /dev/null +++ b/pkg/llm/tasks/dify/dify_batch_create_task.go @@ -0,0 +1,40 @@ +package dify + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/llm/models" +) + +type DifyBatchCreateTask struct { + taskman.STask +} + +func init() { + taskman.RegisterTask(DifyBatchCreateTask{}) +} + +func (task *DifyBatchCreateTask) OnInit(ctx context.Context, objs []db.IStandaloneModel, body jsonutils.JSONObject) { + task.SetStage("OnDifyCreateCompleteAll", nil) + + inputs := make([]api.DifyCreateInput, 0) + task.GetParams().Unmarshal(&inputs, "data") + + for i := range objs { + dify := objs[i].(*models.SDify) + err := dify.StartCreateTask(ctx, task.UserCred, inputs[i], task.GetTaskId()) + if err != nil { + log.Errorf("start task for %d dify %s(%s) fail %s", i, dify.Id, dify.Name, err) + } + } +} + +func (task *DifyBatchCreateTask) OnDifyCreateCompleteAll(ctx context.Context, objs []db.IStandaloneModel, body jsonutils.JSONObject) { + task.SetStageComplete(ctx, nil) +} diff --git a/pkg/llm/tasks/dify/dify_create_task.go b/pkg/llm/tasks/dify/dify_create_task.go new file mode 100644 index 0000000000..b5028213e5 --- /dev/null +++ b/pkg/llm/tasks/dify/dify_create_task.go @@ -0,0 +1,154 @@ +package dify + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/llm/models" + "yunion.io/x/onecloud/pkg/util/logclient" +) + +type DifyCreateTask struct { + taskman.STask +} + +func init() { + taskman.RegisterTask(DifyCreateTask{}) +} + +func (task *DifyCreateTask) taskFailed(ctx context.Context, dify *models.SDify, err error) { + dify.SetStatus(ctx, task.UserCred, api.LLM_STATUS_CREATE_FAIL, err.Error()) + db.OpsLog.LogEvent(dify, db.ACT_CREATE, err, task.UserCred) + logclient.AddActionLogWithStartable(task, dify, logclient.ACT_CREATE, err, task.UserCred, false) + task.SetStageFailed(ctx, jsonutils.NewString(err.Error())) +} + +func (task *DifyCreateTask) taskComplete(ctx context.Context, dify *models.SDify, status string) { + dify.SetStatus(ctx, task.GetUserCred(), status, "create success") + task.SetStageComplete(ctx, nil) +} + +func (task *DifyCreateTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + dify := obj.(*models.SDify) + serverCreateInput := api.DifyCreateInput{} + err := body.Unmarshal(&serverCreateInput) + if err != nil { + task.taskFailed(ctx, dify, err) + return + } + + serverCreateInput.Name = dify.Name + serverId, err := dify.ServerCreate(ctx, task.UserCred, &serverCreateInput) + if err != nil { + task.taskFailed(ctx, dify, err) + return + } + + db.Update(dify, func() error { + dify.SvrId = serverId + return nil + }) + dify.SvrId = serverId + task.SetStage("OnDifyRefreshStatusComplete", nil) + var expectStatus []string + if serverCreateInput.AutoStart { + expectStatus = []string{computeapi.VM_RUNNING} + } else { + expectStatus = []string{computeapi.VM_READY} + } + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + server, err := dify.WaitServerStatus(ctx, task.UserCred, expectStatus, 7200) + if err != nil { + return nil, errors.Wrap(err, "WaitServerStatus") + } + return jsonutils.Marshal(server), nil + }) +} + +func (task *DifyCreateTask) OnDifyRefreshStatusCompleteFailed(ctx context.Context, dify *models.SDify, err jsonutils.JSONObject) { + task.taskFailed(ctx, dify, errors.Error(err.String())) +} + +func (task *DifyCreateTask) OnDifyRefreshStatusComplete(ctx context.Context, dify *models.SDify, body jsonutils.JSONObject) { + server := computeapi.ServerDetails{} + err := body.Unmarshal(&server) + if err != nil { + task.taskFailed(ctx, dify, errors.Wrap(err, "Unmarshal")) + return + } + + // 创建磁盘 + for _, disk := range server.DisksInfo { + volume := models.SVolume{} + volume.SvrId = disk.Id + volume.LLMId = dify.Id + volume.SizeMB = disk.SizeMb + volume.Name = disk.Name + volume.StorageType = disk.StorageType + volume.Status = computeapi.DISK_READY + volume.DomainId = dify.DomainId + volume.ProjectId = dify.ProjectId + volume.ProjectSrc = dify.ProjectSrc + // if len(input.TemplateId) > 0 { + volume.TemplateId = disk.ImageId + // } + // volume.MountedApps = mountedApps + + err := models.GetVolumeManager().TableSpec().Insert(ctx, &volume) + if err != nil { + task.taskFailed(ctx, dify, errors.Wrap(err, "VolumeManager.TableSpec().Insert")) + return + } + } + + // 创建访问信息、portmappings + if len(server.Nics) > 0 { + db.Update(dify, func() error { + dify.LLMIp = server.Nics[0].IpAddr + return nil + }) + + for _, portMapping := range server.Nics[0].PortMappings { + access := models.SAccessInfo{} + access.LLMId = dify.Id + + access.ListenPort = int(portMapping.Port) + access.AccessPort = int(*portMapping.HostPort) + access.Protocol = string(portMapping.Protocol) + access.RemoteIps = portMapping.RemoteIps + envs := make([]api.PortMappingEnv, 0) + for _, env := range portMapping.Envs { + envs = append(envs, api.PortMappingEnv{ + Key: env.Key, + ValueFrom: string(env.ValueFrom), + }) + } + access.PortMappingEnvs = envs + + models.GetAccessInfoManager().TableSpec().Insert(ctx, &access) + } + } + + // // 创建应用容器记录 + // if len(server.Containers) != 1 { + // task.taskFailed(ctx, dify, errors.Errorf("expected 1 containers, but got %d", len(server.Containers))) + // return + // } + // llmCtr := models.GetSvrLLMContainer(server.Containers) + // if llmCtr == nil { + // task.taskFailed(ctx, dify, errors.Errorf("cannot find app container")) + // return + // } + // if _, err := models.GetLLMContainerManager().CreateOnLLM(ctx, task.GetUserCred(), dify.GetOwnerId(), dify, llmCtr.Id, llmCtr.Name); nil != err { + // task.taskFailed(ctx, dify, errors.Wrap(err, "create llm container on llm")) + // return + // } + + task.taskComplete(ctx, dify, server.Status) +} diff --git a/pkg/llm/tasks/dify/dify_delete_task.go b/pkg/llm/tasks/dify/dify_delete_task.go new file mode 100644 index 0000000000..a82bd26042 --- /dev/null +++ b/pkg/llm/tasks/dify/dify_delete_task.go @@ -0,0 +1,87 @@ +package dify + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/llm/models" + "yunion.io/x/onecloud/pkg/util/logclient" +) + +type DifyDeleteTask struct { + taskman.STask +} + +func init() { + taskman.RegisterTask(DifyDeleteTask{}) +} + +func (task *DifyDeleteTask) taskFailed(ctx context.Context, dify *models.SDify, err error) { + dify.SetStatus(ctx, task.UserCred, api.LLM_STATUS_DELETE_FAILED, err.Error()) + db.OpsLog.LogEvent(dify, db.ACT_DELETE_FAIL, err, task.UserCred) + logclient.AddActionLogWithStartable(task, dify, logclient.ACT_DELETE, err, task.UserCred, false) + task.SetStageFailed(ctx, jsonutils.NewString(err.Error())) +} + +func (task *DifyDeleteTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + dify := obj.(*models.SDify) + dify.SetStatus(ctx, task.UserCred, api.LLM_STATUS_DELETING, "start delete") + + if len(dify.SvrId) == 0 { + task.OnDifyRefreshStatusComplete(ctx, dify, nil) + return + } + + err := dify.ServerDelete(ctx, task.UserCred) + if err != nil { + task.taskFailed(ctx, dify, err) + return + } + task.SetStage("OnDifyRefreshStatusComplete", nil) + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + err = dify.WaitDelete(ctx, task.UserCred, 1800) + if err != nil { + return nil, errors.Wrap(err, "llm.WaitDelete") + } + + return nil, nil + }) +} + +func (task *DifyDeleteTask) OnDifyRefreshStatusCompleteFailed(ctx context.Context, dify *models.SDify, err jsonutils.JSONObject) { + task.taskFailed(ctx, dify, errors.Error(err.String())) +} + +func (task *DifyDeleteTask) OnDifyRefreshStatusComplete(ctx context.Context, dify *models.SDify, body jsonutils.JSONObject) { + volume, err := dify.GetVolume() + if err != nil { + if errors.Cause(err) != errors.ErrNotFound { + task.taskFailed(ctx, dify, err) + return + } + } + if volume != nil { + task.SetStage("OnDifyVolumeDeleteComplete", nil) + volume.StartDeleteTask(ctx, task.UserCred, task.GetTaskId()) + } else { + task.OnDifyVolumeDeleteComplete(ctx, dify, nil) + } +} + +func (task *DifyDeleteTask) OnDifyVolumeDeleteCompleteFailed(ctx context.Context, dify *models.SDify, err jsonutils.JSONObject) { + task.taskFailed(ctx, dify, errors.Error(err.String())) +} + +func (task *DifyDeleteTask) OnDifyVolumeDeleteComplete(ctx context.Context, dify *models.SDify, body jsonutils.JSONObject) { + err := dify.RealDelete(ctx, task.UserCred) + if err != nil { + task.taskFailed(ctx, dify, err) + return + } + task.SetStageComplete(ctx, nil) +} diff --git a/pkg/llm/tasks/dify/dify_start_task.go b/pkg/llm/tasks/dify/dify_start_task.go new file mode 100644 index 0000000000..2ebc9def4e --- /dev/null +++ b/pkg/llm/tasks/dify/dify_start_task.go @@ -0,0 +1,80 @@ +package dify + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/llm/models" + "yunion.io/x/onecloud/pkg/llm/options" + "yunion.io/x/onecloud/pkg/llm/tasks/worker" + "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/mcclient/modules/compute" + "yunion.io/x/onecloud/pkg/util/logclient" +) + +type DifyStartTask struct { + taskman.STask +} + +func init() { + taskman.RegisterTask(DifyStartTask{}) +} + +func (task *DifyStartTask) taskFailed(ctx context.Context, dify *models.SDify, err string) { + dify.SetStatus(ctx, task.UserCred, api.LLM_STATUS_START_FAIL, err) + db.OpsLog.LogEvent(dify, db.ACT_START, err, task.UserCred) + logclient.AddActionLogWithStartable(task, dify, logclient.ACT_START, err, task.UserCred, false) + // llm.NotifyRequest(ctx, task.GetUserCred(), notify.ActionStart, nil, false) + task.SetStageFailed(ctx, jsonutils.NewString(err)) +} + +func (task *DifyStartTask) taskComplete(ctx context.Context, dify *models.SDify) { + dify.SetStatus(ctx, task.GetUserCred(), api.LLM_STATUS_RUNNING, "start complete") + // llm.NotifyRequest(ctx, task.GetUserCred(), notify.ActionStart, nil, true) + task.SetStageComplete(ctx, nil) +} + +func (t *DifyStartTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + t.requestStart(ctx, obj.(*models.SDify)) +} + +func (t *DifyStartTask) requestStart(ctx context.Context, dify *models.SDify) { + s := auth.GetSession(ctx, t.GetUserCred(), options.Options.Region) + _, err := compute.Servers.PerformAction(s, dify.SvrId, "start", nil) + if err != nil { + t.taskFailed(ctx, dify, err.Error()) + return + } + + t.SetStage("OnStarted", nil) + worker.StartTaskRun(t, func() (jsonutils.JSONObject, error) { + _, err := dify.WaitServerStatus(ctx, t.GetUserCred(), []string{computeapi.VM_RUNNING}, 900) + if err != nil { + return nil, errors.Wrap(err, "WaitServerStatus") + } + // time.Sleep(time.Second) + // _, err = d.WaitServerStatus(ctx, task.UserCred, []string{computeapi.VM_RUNNING}, 900) + // if err != nil { + // return nil, errors.Wrap(err, "WaitServerStatus") + // } + return nil, nil + }) + // if err := llm.RunModel(ctx, t.GetUserCred()); nil != err { + // t.OnStartedFailed(ctx, llm, jsonutils.NewString(err.Error())) + // return + // } +} + +func (t *DifyStartTask) OnStartedFailed(ctx context.Context, dify *models.SDify, err jsonutils.JSONObject) { + t.taskFailed(ctx, dify, err.String()) +} + +func (t *DifyStartTask) OnStarted(ctx context.Context, dify *models.SDify, reason jsonutils.JSONObject) { + t.taskComplete(ctx, dify) +} diff --git a/pkg/llm/tasks/dify/dify_stop_task.go b/pkg/llm/tasks/dify/dify_stop_task.go new file mode 100644 index 0000000000..957d8b58c6 --- /dev/null +++ b/pkg/llm/tasks/dify/dify_stop_task.go @@ -0,0 +1,96 @@ +package dify + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/llm/models" + "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/mcclient/modules/compute" + "yunion.io/x/onecloud/pkg/util/logclient" +) + +type DifyStopTask struct { + taskman.STask +} + +func init() { + taskman.RegisterTask(DifyStopTask{}) +} + +func (task *DifyStopTask) taskFailed(ctx context.Context, dify *models.SDify, err string) { + dify.SetStatus(ctx, task.UserCred, api.LLM_STATUS_STOP_FAILED, err) + db.OpsLog.LogEvent(dify, db.ACT_STOP, err, task.UserCred) + logclient.AddActionLogWithStartable(task, dify, logclient.ACT_VM_STOP, err, task.UserCred, false) + // llm.NotifyRequest(ctx, task.GetUserCred(), notify.ActionStop, nil, false) + task.SetStageFailed(ctx, jsonutils.NewString(err)) +} + +func (task *DifyStopTask) taskComplete(ctx context.Context, dify *models.SDify) { + if !task.HasParentTask() { + dify.SetStatus(ctx, task.GetUserCred(), api.LLM_STATUS_READY, "") + } + // llm.NotifyRequest(ctx, task.GetUserCred(), notify.ActionStop, nil, true) + task.SetStageComplete(ctx, nil) +} + +func (task *DifyStopTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + dify := obj.(*models.SDify) + + srv, err := dify.GetServer(ctx) + if err != nil { + task.taskFailed(ctx, dify, errors.Wrap(err, "GetServer").Error()) + return + } + + if srv.Status == computeapi.VM_READY { + task.taskComplete(ctx, dify) + return + } + + task.SetStage("OnStopComplete", nil) + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + s := auth.GetSession(ctx, task.UserCred, "") + _, err = compute.Servers.PerformAction(s, dify.SvrId, "stop", nil) + if err != nil { + task.taskFailed(ctx, dify, err.Error()) + return nil, errors.Wrap(err, "server perform stop") + } + _, err := dify.WaitServerStatus(ctx, task.UserCred, []string{computeapi.VM_READY}, 600) + if err != nil { + if errors.Cause(err) == errors.ErrTimeout { + params := computeapi.ServerStopInput{ + IsForce: true, + TimeoutSecs: 10, + } + _, err = compute.Servers.PerformAction(s, dify.SvrId, "stop", jsonutils.Marshal(params)) + if err != nil { + return nil, errors.Wrap(err, "server perform stop by force") + } + _, err := dify.WaitServerStatus(ctx, task.UserCred, []string{computeapi.VM_READY}, 600) + if err != nil { + return nil, errors.Wrap(err, "WaitServerStatus 2") + } + } else { + return nil, errors.Wrap(err, "WaitServerStatus") + } + } + return nil, nil + }) +} + +func (task *DifyStopTask) OnStopComplete(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + dify := obj.(*models.SDify) + task.taskComplete(ctx, dify) +} + +func (task *DifyStopTask) OnStopCompleteFailed(ctx context.Context, obj db.IStandaloneModel, err jsonutils.JSONObject) { + dify := obj.(*models.SDify) + task.taskFailed(ctx, dify, err.String()) +} diff --git a/pkg/llm/tasks/dify/doc.go b/pkg/llm/tasks/dify/doc.go new file mode 100644 index 0000000000..651b9d1ee6 --- /dev/null +++ b/pkg/llm/tasks/dify/doc.go @@ -0,0 +1 @@ +package dify // import "yunion.io/x/onecloud/pkg/llm/tasks/dify" diff --git a/pkg/llm/tasks/doc.go b/pkg/llm/tasks/doc.go new file mode 100644 index 0000000000..30368d3720 --- /dev/null +++ b/pkg/llm/tasks/doc.go @@ -0,0 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tasks // import "yunion.io/x/onecloud/pkg/llm/tasks" diff --git a/pkg/llm/tasks/llm/doc.go b/pkg/llm/tasks/llm/doc.go new file mode 100644 index 0000000000..d6c799b547 --- /dev/null +++ b/pkg/llm/tasks/llm/doc.go @@ -0,0 +1 @@ +package llm // import "yunion.io/x/onecloud/pkg/llm/tasks/llm" diff --git a/pkg/llm/tasks/llm/llm_batch_create_task.go b/pkg/llm/tasks/llm/llm_batch_create_task.go new file mode 100644 index 0000000000..bd034030e8 --- /dev/null +++ b/pkg/llm/tasks/llm/llm_batch_create_task.go @@ -0,0 +1,40 @@ +package llm + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/llm/models" +) + +type LLMBatchCreateTask struct { + taskman.STask +} + +func init() { + taskman.RegisterTask(LLMBatchCreateTask{}) +} + +func (task *LLMBatchCreateTask) OnInit(ctx context.Context, objs []db.IStandaloneModel, body jsonutils.JSONObject) { + task.SetStage("OnLLMCreateCompleteAll", nil) + + inputs := make([]api.LLMCreateInput, 0) + task.GetParams().Unmarshal(&inputs, "data") + + for i := range objs { + llm := objs[i].(*models.SLLM) + err := llm.StartCreateTask(ctx, task.UserCred, inputs[i], task.GetTaskId()) + if err != nil { + log.Errorf("start task for %d llm %s(%s) fail %s", i, llm.Id, llm.Name, err) + } + } +} + +func (task *LLMBatchCreateTask) OnLLMCreateCompleteAll(ctx context.Context, objs []db.IStandaloneModel, body jsonutils.JSONObject) { + task.SetStageComplete(ctx, nil) +} diff --git a/pkg/llm/tasks/llm/llm_create_task.go b/pkg/llm/tasks/llm/llm_create_task.go new file mode 100644 index 0000000000..884c7f29e9 --- /dev/null +++ b/pkg/llm/tasks/llm/llm_create_task.go @@ -0,0 +1,172 @@ +package llm + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/llm/models" + "yunion.io/x/onecloud/pkg/util/logclient" +) + +type LLMCreateTask struct { + taskman.STask +} + +func init() { + taskman.RegisterTask(LLMCreateTask{}) +} + +func (task *LLMCreateTask) taskFailed(ctx context.Context, llm *models.SLLM, err error) { + llm.SetStatus(ctx, task.UserCred, api.LLM_STATUS_CREATE_FAIL, err.Error()) + db.OpsLog.LogEvent(llm, db.ACT_CREATE, err, task.UserCred) + logclient.AddActionLogWithStartable(task, llm, logclient.ACT_CREATE, err, task.UserCred, false) + task.SetStageFailed(ctx, jsonutils.NewString(err.Error())) +} + +func (task *LLMCreateTask) taskComplete(ctx context.Context, llm *models.SLLM, status string) { + llm.SetStatus(ctx, task.GetUserCred(), status, "create success") + task.SetStageComplete(ctx, nil) +} + +func (task *LLMCreateTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + llm := obj.(*models.SLLM) + serverCreateInput := api.LLMCreateInput{} + err := body.Unmarshal(&serverCreateInput) + if err != nil { + task.taskFailed(ctx, llm, err) + return + } + + serverCreateInput.Name = llm.Name + serverId, err := llm.ServerCreate(ctx, task.UserCred, &serverCreateInput) + if err != nil { + task.taskFailed(ctx, llm, err) + return + } + + db.Update(llm, func() error { + llm.SvrId = serverId + return nil + }) + llm.SvrId = serverId + task.SetStage("OnLLMRefreshStatusComplete", nil) + var expectStatus []string + if serverCreateInput.AutoStart { + expectStatus = []string{computeapi.VM_RUNNING} + } else { + expectStatus = []string{computeapi.VM_READY} + } + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + server, err := llm.WaitServerStatus(ctx, task.UserCred, expectStatus, 7200) + if err != nil { + return nil, errors.Wrap(err, "WaitServerStatus") + } + return jsonutils.Marshal(server), nil + }) +} + +func (task *LLMCreateTask) OnLLMRefreshStatusCompleteFailed(ctx context.Context, llm *models.SLLM, err jsonutils.JSONObject) { + task.taskFailed(ctx, llm, errors.Error(err.String())) +} + +func (task *LLMCreateTask) OnLLMRefreshStatusComplete(ctx context.Context, llm *models.SLLM, body jsonutils.JSONObject) { + server := computeapi.ServerDetails{} + err := body.Unmarshal(&server) + if err != nil { + task.taskFailed(ctx, llm, errors.Wrap(err, "Unmarshal")) + return + } + + // 创建磁盘 + for _, disk := range server.DisksInfo { + volume := models.SVolume{} + volume.SvrId = disk.Id + volume.LLMId = llm.Id + volume.SizeMB = disk.SizeMb + volume.Name = disk.Name + volume.StorageType = disk.StorageType + volume.Status = computeapi.DISK_READY + volume.DomainId = llm.DomainId + volume.ProjectId = llm.ProjectId + volume.ProjectSrc = llm.ProjectSrc + // if len(input.TemplateId) > 0 { + volume.TemplateId = disk.ImageId + // } + // volume.MountedApps = mountedApps + + err := models.GetVolumeManager().TableSpec().Insert(ctx, &volume) + if err != nil { + task.taskFailed(ctx, llm, errors.Wrap(err, "VolumeManager.TableSpec().Insert")) + return + } + } + + // 创建访问信息、portmappings + if len(server.Nics) > 0 { + db.Update(llm, func() error { + llm.LLMIp = server.Nics[0].IpAddr + return nil + }) + + for _, portMapping := range server.Nics[0].PortMappings { + access := models.SAccessInfo{} + access.LLMId = llm.Id + + access.ListenPort = int(portMapping.Port) + access.AccessPort = int(*portMapping.HostPort) + access.Protocol = string(portMapping.Protocol) + access.RemoteIps = portMapping.RemoteIps + envs := make([]api.PortMappingEnv, 0) + for _, env := range portMapping.Envs { + envs = append(envs, api.PortMappingEnv{ + Key: env.Key, + ValueFrom: string(env.ValueFrom), + }) + } + access.PortMappingEnvs = envs + + models.GetAccessInfoManager().TableSpec().Insert(ctx, &access) + } + } + + // 创建应用容器记录 + if len(server.Containers) != 1 { + task.taskFailed(ctx, llm, errors.Errorf("expected 1 containers, but got %d", len(server.Containers))) + return + } + llmCtr := models.GetSvrLLMContainer(server.Containers) + if llmCtr == nil { + task.taskFailed(ctx, llm, errors.Errorf("cannot find app container")) + return + } + if _, err := models.GetLLMContainerManager().CreateOnLLM(ctx, task.GetUserCred(), llm.GetOwnerId(), llm, llmCtr.Id, llmCtr.Name); nil != err { + task.taskFailed(ctx, llm, errors.Wrap(err, "create llm container on llm")) + return + } + + task.taskComplete(ctx, llm, server.Status) + // // 调用子任务在容器中拉取模型 + // params := jsonutils.NewDict() + // params.Set("status", jsonutils.NewString(server.Status)) + // task.SetStage("OnLLMPullModel", params) + + // if err := llm.StartPullModelTask(ctx, task.GetUserCred(), nil, task.GetId()); err != nil { + // task.taskFailed(ctx, llm, errors.Wrap(err, "StartPullModelTask")) + // return + // } +} + +// func (task *LLMCreateTask) OnLLMPullModelFailed(ctx context.Context, llm *models.SLLM, err jsonutils.JSONObject) { +// task.taskFailed(ctx, llm, errors.Error(err.String())) +// } + +// func (task *LLMCreateTask) OnLLMPullModel(ctx context.Context, llm *models.SLLM, body jsonutils.JSONObject) { +// status, _ := task.GetParams().GetString("status") +// task.taskComplete(ctx, llm, status) +// } diff --git a/pkg/llm/tasks/llm/llm_delete_task.go b/pkg/llm/tasks/llm/llm_delete_task.go new file mode 100644 index 0000000000..76ec0fc371 --- /dev/null +++ b/pkg/llm/tasks/llm/llm_delete_task.go @@ -0,0 +1,108 @@ +package llm + +import ( + "context" + "database/sql" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/llm/models" + "yunion.io/x/onecloud/pkg/util/logclient" +) + +type LLMDeleteTask struct { + taskman.STask +} + +func init() { + taskman.RegisterTask(LLMDeleteTask{}) +} + +func (task *LLMDeleteTask) taskFailed(ctx context.Context, llm *models.SLLM, err error) { + llm.SetStatus(ctx, task.UserCred, api.LLM_STATUS_DELETE_FAILED, err.Error()) + db.OpsLog.LogEvent(llm, db.ACT_DELETE_FAIL, err, task.UserCred) + logclient.AddActionLogWithStartable(task, llm, logclient.ACT_DELETE, err, task.UserCred, false) + task.SetStageFailed(ctx, jsonutils.NewString(err.Error())) +} + +func (task *LLMDeleteTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + llm := obj.(*models.SLLM) + llm.SetStatus(ctx, task.UserCred, api.LLM_STATUS_DELETING, "start delete") + + if len(llm.SvrId) == 0 { + task.OnLLMRefreshStatusComplete(ctx, llm, nil) + return + } + + err := llm.ServerDelete(ctx, task.UserCred) + if err != nil { + task.taskFailed(ctx, llm, err) + return + } + task.SetStage("OnLLMRefreshStatusComplete", nil) + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + err = llm.WaitDelete(ctx, task.UserCred, 1800) + if err != nil { + return nil, errors.Wrap(err, "llm.WaitDelete") + } + + return nil, nil + }) +} + +func (task *LLMDeleteTask) OnLLMRefreshStatusCompleteFailed(ctx context.Context, llm *models.SLLM, err jsonutils.JSONObject) { + task.taskFailed(ctx, llm, errors.Error(err.String())) +} + +func (task *LLMDeleteTask) OnLLMRefreshStatusComplete(ctx context.Context, llm *models.SLLM, body jsonutils.JSONObject) { + volume, err := llm.GetVolume() + if err != nil { + if errors.Cause(err) != errors.ErrNotFound { + task.taskFailed(ctx, llm, err) + return + } + } + if volume != nil { + task.SetStage("OnLLMVolumeDeleteComplete", nil) + volume.StartDeleteTask(ctx, task.UserCred, task.GetTaskId()) + } else { + task.OnLLMVolumeDeleteComplete(ctx, llm, nil) + } +} + +func (task *LLMDeleteTask) OnLLMVolumeDeleteCompleteFailed(ctx context.Context, llm *models.SLLM, err jsonutils.JSONObject) { + task.taskFailed(ctx, llm, errors.Error(err.String())) +} + +func (task *LLMDeleteTask) OnLLMVolumeDeleteComplete(ctx context.Context, llm *models.SLLM, body jsonutils.JSONObject) { + lc, err := llm.GetLLMContainer() + if err != nil { + if errors.Cause(err) != errors.ErrNotFound && errors.Cause(err) != sql.ErrNoRows { + task.taskFailed(ctx, llm, err) + return + } + } + if lc != nil { + task.SetStage("OnLLMContainerDeleteComplete", nil) + lc.StartDeleteTask(ctx, task.UserCred, task.GetTaskId()) + } else { + task.OnLLMContainerDeleteComplete(ctx, llm, nil) + } +} + +func (task *LLMDeleteTask) OnLLMContainerDeleteCompleteFailed(ctx context.Context, llm *models.SLLM, err jsonutils.JSONObject) { + task.taskFailed(ctx, llm, errors.Error(err.String())) +} + +func (task *LLMDeleteTask) OnLLMContainerDeleteComplete(ctx context.Context, llm *models.SLLM, body jsonutils.JSONObject) { + err := llm.RealDelete(ctx, task.UserCred) + if err != nil { + task.taskFailed(ctx, llm, err) + return + } + task.SetStageComplete(ctx, nil) +} diff --git a/pkg/llm/tasks/llm/llm_install_gguf_file_task.go b/pkg/llm/tasks/llm/llm_install_gguf_file_task.go new file mode 100644 index 0000000000..3fc9b5d1ab --- /dev/null +++ b/pkg/llm/tasks/llm/llm_install_gguf_file_task.go @@ -0,0 +1,85 @@ +package llm + +// import ( +// "context" + +// "yunion.io/x/jsonutils" + +// api "yunion.io/x/onecloud/pkg/apis/llm" +// "yunion.io/x/onecloud/pkg/cloudcommon/db" +// "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" +// "yunion.io/x/onecloud/pkg/llm/models" +// ) + +// type LLMInstallGgufTask struct { +// LLMBaseTask +// } + +// func init() { +// taskman.RegisterTask(LLMInstallGgufTask{}) +// } + +// func (t *LLMInstallGgufTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { +// if err := obj.(*models.SOllama).ConfirmContainerId(ctx, t.GetUserCred()); err != nil { +// t.OnGetGgufFileFailed(ctx, obj.(*models.SOllama), jsonutils.NewString(err.Error())) +// } +// t.requestGetGgufFile(ctx, obj.(*models.SOllama)) +// } + +// func (t *LLMInstallGgufTask) requestGetGgufFile(ctx context.Context, llm *models.SOllama) { +// // Update status +// llm.SetStatus(ctx, t.GetUserCred(), api.LLM_STATUS_FETCHING_GGUF_FILE, "") + +// // Distingush whether access gguf file from host or download from web +// input := new(api.OllamaGgufSpec) +// if err := t.GetParams().Unmarshal(input); nil != err { +// t.OnGetGgufFileFailed(ctx, llm, jsonutils.NewString(err.Error())) +// return +// } + +// switch input.Source { +// case api.LLM_OLLAMA_GGUF_SOURCE_WEB: +// t.SetStage("OnGetGgufFile", nil) +// if err := llm.DownloadGgufFile(ctx, t.GetUserCred(), t.GetId()); nil != err { +// t.OnGetGgufFileFailed(ctx, llm, jsonutils.NewString(err.Error())) +// return +// } +// default: +// if err := llm.AccessGgufFile(ctx, t.GetUserCred(), t); nil != err { +// t.OnGetGgufFileFailed(ctx, llm, jsonutils.NewString(err.Error())) +// return +// } +// t.OnGetGgufFile(ctx, llm, nil) +// } +// } + +// func (t *LLMInstallGgufTask) OnGetGgufFileFailed(ctx context.Context, llm *models.SOllama, reason jsonutils.JSONObject) { +// llm.SetStatus(ctx, t.GetUserCred(), api.LLM_STATUS_FETCH_GGUF_FILE_FAILED, reason.String()) +// t.SetStageFailed(ctx, reason) +// } + +// func (t *LLMInstallGgufTask) OnGetGgufFile(ctx context.Context, llm *models.SOllama, data jsonutils.JSONObject) { +// input := new(api.OllamaGgufSpec) +// if err := t.GetParams().Unmarshal(input); nil != err { +// t.OnCreateModelFailed(ctx, llm, jsonutils.NewString(err.Error())) +// return +// } + +// llm.SetStatus(ctx, t.GetUserCred(), api.LLM_STATUS_CREATING_GGUF_MODEL, "") + +// if err := llm.InstallGgufModel(ctx, t.GetUserCred(), input.ModelFile); nil != err { +// t.OnCreateModelFailed(ctx, llm, jsonutils.NewString(err.Error())) +// return +// } +// t.OnPulledModel(ctx, llm, nil) +// } + +// func (t *LLMInstallGgufTask) OnCreateModelFailed(ctx context.Context, llm *models.SOllama, reason jsonutils.JSONObject) { +// llm.SetStatus(ctx, t.GetUserCred(), api.LLM_STATUS_CREATE_GGUF_MODEL_FAILED, reason.String()) +// t.SetStageFailed(ctx, reason) +// } + +// func (t *LLMInstallGgufTask) OnPulledModel(ctx context.Context, llm *models.SOllama, data jsonutils.JSONObject) { +// llm.SetStatus(ctx, t.GetUserCred(), api.LLM_STATUS_PULLED_MODEL, "") +// t.SetStageComplete(ctx, nil) +// } diff --git a/pkg/llm/tasks/llm/llm_pull_model_task.go b/pkg/llm/tasks/llm/llm_pull_model_task.go new file mode 100644 index 0000000000..406e85eb09 --- /dev/null +++ b/pkg/llm/tasks/llm/llm_pull_model_task.go @@ -0,0 +1,74 @@ +package llm + +// import ( +// "context" + +// "yunion.io/x/jsonutils" + +// api "yunion.io/x/onecloud/pkg/apis/llm" +// "yunion.io/x/onecloud/pkg/cloudcommon/db" +// "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" +// "yunion.io/x/onecloud/pkg/llm/models" +// ) + +// type LLMPullModelTask struct { +// taskman.STask +// } + +// func (t *LLMPullModelTask) GetLLM() *models.SLLM { +// return t.GetObject().(*models.SLLM) +// } + +// func (t *LLMPullModelTask) GetLLMContainerDriver() models.ILLMContainerDriver { +// return t.GetLLM().GetLLMContainerDriver() +// } + +// func init() { +// taskman.RegisterTask(LLMPullModelTask{}) +// } + +// func (t *LLMPullModelTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { +// t.requestGetManifests(ctx, obj.(*models.SLLM)) +// } + +// func (t *LLMPullModelTask) requestGetManifests(ctx context.Context, llm *models.SLLM) { +// // llm.SetStatus(ctx, t.GetUserCred(), api.LLM_STATUS_PULLING_MODEL, "") +// t.SetStage("OnGetManifests", nil) + +// if err := t.GetLLMContainerDriver().GetManifests(ctx, t.GetUserCred(), llm, t.GetId()); err != nil { +// t.OnGetManifestsFailed(ctx, llm, jsonutils.NewString(err.Error())) +// return +// } +// } + +// func (t *LLMPullModelTask) OnGetManifestsFailed(ctx context.Context, llm *models.SLLM, reason jsonutils.JSONObject) { +// llm.SetStatus(ctx, t.GetUserCred(), api.LLM_STATUS_GET_MANIFESTS_FAILED, reason.String()) +// t.SetStageFailed(ctx, reason) +// } + +// func (t *LLMPullModelTask) OnGetManifests(ctx context.Context, llm *models.SLLM, data jsonutils.JSONObject) { +// t.SetStage("OnAccessCache", nil) +// if err := t.GetLLMContainerDriver().AccessBlobsCache(ctx, t.GetUserCred(), llm, t.GetId()); nil != err { +// t.OnAccessCacheFailed(ctx, llm, jsonutils.NewString(err.Error())) +// return +// } +// } + +// func (t *LLMPullModelTask) OnAccessCacheFailed(ctx context.Context, llm *models.SLLM, reason jsonutils.JSONObject) { +// llm.SetStatus(ctx, t.GetUserCred(), api.LLM_STATUS_DOWNLOADING_BLOBS_FAILED, reason.String()) +// t.SetStageFailed(ctx, reason) +// } + +// func (t *LLMPullModelTask) OnAccessCache(ctx context.Context, llm *models.SLLM, data jsonutils.JSONObject) { +// // log.Infoln("try to find out blobs: ", t.GetParams().String(), data.String()) +// if err := t.GetLLMContainerDriver().CopyBlobs(ctx, t.GetUserCred(), llm); nil != err { +// t.OnAccessCacheFailed(ctx, llm, jsonutils.NewString(err.Error())) +// return +// } +// t.OnPulledModel(ctx, llm, nil) +// } + +// func (t *LLMPullModelTask) OnPulledModel(ctx context.Context, llm *models.SLLM, data jsonutils.JSONObject) { +// // llm.SetStatus(ctx, t.GetUserCred(), api.LLM_STATUS_PULLED_MODEL, "") +// t.SetStageComplete(ctx, nil) +// } diff --git a/pkg/llm/tasks/llm/llm_start_task.go b/pkg/llm/tasks/llm/llm_start_task.go new file mode 100644 index 0000000000..ebc052671d --- /dev/null +++ b/pkg/llm/tasks/llm/llm_start_task.go @@ -0,0 +1,80 @@ +package llm + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/llm/models" + "yunion.io/x/onecloud/pkg/llm/options" + "yunion.io/x/onecloud/pkg/llm/tasks/worker" + "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/mcclient/modules/compute" + "yunion.io/x/onecloud/pkg/util/logclient" +) + +type LLMStartTask struct { + taskman.STask +} + +func init() { + taskman.RegisterTask(LLMStartTask{}) +} + +func (task *LLMStartTask) taskFailed(ctx context.Context, llm *models.SLLM, err string) { + llm.SetStatus(ctx, task.UserCred, api.LLM_STATUS_START_FAIL, err) + db.OpsLog.LogEvent(llm, db.ACT_START, err, task.UserCred) + logclient.AddActionLogWithStartable(task, llm, logclient.ACT_START, err, task.UserCred, false) + // llm.NotifyRequest(ctx, task.GetUserCred(), notify.ActionStart, nil, false) + task.SetStageFailed(ctx, jsonutils.NewString(err)) +} + +func (task *LLMStartTask) taskComplete(ctx context.Context, llm *models.SLLM) { + llm.SetStatus(ctx, task.GetUserCred(), api.LLM_STATUS_RUNNING, "start complete") + // llm.NotifyRequest(ctx, task.GetUserCred(), notify.ActionStart, nil, true) + task.SetStageComplete(ctx, nil) +} + +func (t *LLMStartTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + t.requestStart(ctx, obj.(*models.SLLM)) +} + +func (t *LLMStartTask) requestStart(ctx context.Context, llm *models.SLLM) { + s := auth.GetSession(ctx, t.GetUserCred(), options.Options.Region) + _, err := compute.Servers.PerformAction(s, llm.SvrId, "start", nil) + if err != nil { + t.taskFailed(ctx, llm, err.Error()) + return + } + + t.SetStage("OnStarted", nil) + worker.StartTaskRun(t, func() (jsonutils.JSONObject, error) { + _, err := llm.WaitServerStatus(ctx, t.GetUserCred(), []string{computeapi.VM_RUNNING}, 900) + if err != nil { + return nil, errors.Wrap(err, "WaitServerStatus") + } + // time.Sleep(time.Second) + // _, err = d.WaitServerStatus(ctx, task.UserCred, []string{computeapi.VM_RUNNING}, 900) + // if err != nil { + // return nil, errors.Wrap(err, "WaitServerStatus") + // } + return nil, nil + }) + // if err := llm.RunModel(ctx, t.GetUserCred()); nil != err { + // t.OnStartedFailed(ctx, llm, jsonutils.NewString(err.Error())) + // return + // } +} + +func (t *LLMStartTask) OnStartedFailed(ctx context.Context, llm *models.SLLM, err jsonutils.JSONObject) { + t.taskFailed(ctx, llm, err.String()) +} + +func (t *LLMStartTask) OnStarted(ctx context.Context, llm *models.SLLM, reason jsonutils.JSONObject) { + t.taskComplete(ctx, llm) +} diff --git a/pkg/llm/tasks/llm/llm_stop_task.go b/pkg/llm/tasks/llm/llm_stop_task.go new file mode 100644 index 0000000000..9bad225a68 --- /dev/null +++ b/pkg/llm/tasks/llm/llm_stop_task.go @@ -0,0 +1,96 @@ +package llm + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/llm/models" + "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/mcclient/modules/compute" + "yunion.io/x/onecloud/pkg/util/logclient" +) + +type LLMStopTask struct { + taskman.STask +} + +func init() { + taskman.RegisterTask(LLMStopTask{}) +} + +func (task *LLMStopTask) taskFailed(ctx context.Context, llm *models.SLLM, err string) { + llm.SetStatus(ctx, task.UserCred, api.LLM_STATUS_STOP_FAILED, err) + db.OpsLog.LogEvent(llm, db.ACT_STOP, err, task.UserCred) + logclient.AddActionLogWithStartable(task, llm, logclient.ACT_VM_STOP, err, task.UserCred, false) + // llm.NotifyRequest(ctx, task.GetUserCred(), notify.ActionStop, nil, false) + task.SetStageFailed(ctx, jsonutils.NewString(err)) +} + +func (task *LLMStopTask) taskComplete(ctx context.Context, llm *models.SLLM) { + if !task.HasParentTask() { + llm.SetStatus(ctx, task.GetUserCred(), api.LLM_STATUS_READY, "") + } + // llm.NotifyRequest(ctx, task.GetUserCred(), notify.ActionStop, nil, true) + task.SetStageComplete(ctx, nil) +} + +func (task *LLMStopTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + llm := obj.(*models.SLLM) + + srv, err := llm.GetServer(ctx) + if err != nil { + task.taskFailed(ctx, llm, errors.Wrap(err, "GetServer").Error()) + return + } + + if srv.Status == computeapi.VM_READY { + task.taskComplete(ctx, llm) + return + } + + task.SetStage("OnStopComplete", nil) + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + s := auth.GetSession(ctx, task.UserCred, "") + _, err = compute.Servers.PerformAction(s, llm.SvrId, "stop", nil) + if err != nil { + task.taskFailed(ctx, llm, err.Error()) + return nil, errors.Wrap(err, "server perform stop") + } + _, err := llm.WaitServerStatus(ctx, task.UserCred, []string{computeapi.VM_READY}, 600) + if err != nil { + if errors.Cause(err) == errors.ErrTimeout { + params := computeapi.ServerStopInput{ + IsForce: true, + TimeoutSecs: 10, + } + _, err = compute.Servers.PerformAction(s, llm.SvrId, "stop", jsonutils.Marshal(params)) + if err != nil { + return nil, errors.Wrap(err, "server perform stop by force") + } + _, err := llm.WaitServerStatus(ctx, task.UserCred, []string{computeapi.VM_READY}, 600) + if err != nil { + return nil, errors.Wrap(err, "WaitServerStatus 2") + } + } else { + return nil, errors.Wrap(err, "WaitServerStatus") + } + } + return nil, nil + }) +} + +func (task *LLMStopTask) OnStopComplete(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + llm := obj.(*models.SLLM) + task.taskComplete(ctx, llm) +} + +func (task *LLMStopTask) OnStopCompleteFailed(ctx context.Context, obj db.IStandaloneModel, err jsonutils.JSONObject) { + llm := obj.(*models.SLLM) + task.taskFailed(ctx, llm, err.String()) +} diff --git a/pkg/llm/tasks/llm_container/doc.go b/pkg/llm/tasks/llm_container/doc.go new file mode 100644 index 0000000000..618f2bedcc --- /dev/null +++ b/pkg/llm/tasks/llm_container/doc.go @@ -0,0 +1 @@ +package llm_container // import "yunion.io/x/onecloud/pkg/llm/tasks/llm_container" diff --git a/pkg/llm/tasks/llm_container/llm_container_delete_task.go b/pkg/llm/tasks/llm_container/llm_container_delete_task.go new file mode 100644 index 0000000000..4b494a33b2 --- /dev/null +++ b/pkg/llm/tasks/llm_container/llm_container_delete_task.go @@ -0,0 +1,37 @@ +package llm_container + +import ( + "context" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/llm/models" + "yunion.io/x/onecloud/pkg/util/logclient" +) + +type LLMContainerDeleteTask struct { + taskman.STask +} + +func init() { + taskman.RegisterTask(LLMContainerDeleteTask{}) +} + +func (task *LLMContainerDeleteTask) taskFailed(ctx context.Context, lc *models.SLLMContainer, status string, err error) { + lc.SetStatus(ctx, task.UserCred, status, err.Error()) + db.OpsLog.LogEvent(lc, db.ACT_DELETE, err, task.UserCred) + logclient.AddActionLogWithStartable(task, lc, logclient.ACT_DELETE, err, task.UserCred, false) + task.SetStageFailed(ctx, jsonutils.NewString(err.Error())) +} + +func (task *LLMContainerDeleteTask) taskComplete(ctx context.Context, lc *models.SLLMContainer) { + lc.RealDelete(ctx, task.GetUserCred()) + task.SetStageComplete(ctx, nil) +} + +func (task *LLMContainerDeleteTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + lc := obj.(*models.SLLMContainer) + task.taskComplete(ctx, lc) +} diff --git a/pkg/llm/tasks/loader.go b/pkg/llm/tasks/loader.go new file mode 100644 index 0000000000..e8f6251569 --- /dev/null +++ b/pkg/llm/tasks/loader.go @@ -0,0 +1,22 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tasks + +import ( + _ "yunion.io/x/onecloud/pkg/llm/tasks/dify" + _ "yunion.io/x/onecloud/pkg/llm/tasks/llm" + _ "yunion.io/x/onecloud/pkg/llm/tasks/llm_container" + _ "yunion.io/x/onecloud/pkg/llm/tasks/volume" +) diff --git a/pkg/llm/tasks/volume/doc.go b/pkg/llm/tasks/volume/doc.go new file mode 100644 index 0000000000..6b0a3fe049 --- /dev/null +++ b/pkg/llm/tasks/volume/doc.go @@ -0,0 +1 @@ +package volume // import "yunion.io/x/onecloud/pkg/llm/tasks/volume" diff --git a/pkg/llm/tasks/volume/volume_delete_task.go b/pkg/llm/tasks/volume/volume_delete_task.go new file mode 100644 index 0000000000..d07ff5c601 --- /dev/null +++ b/pkg/llm/tasks/volume/volume_delete_task.go @@ -0,0 +1,70 @@ +package volume + +import ( + "context" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/httputils" + + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/llm/models" + "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/mcclient/modules/compute" + "yunion.io/x/onecloud/pkg/util/logclient" +) + +type VolumeDeleteTask struct { + taskman.STask +} + +func init() { + taskman.RegisterTask(VolumeDeleteTask{}) +} + +func (volumeDeleteTask *VolumeDeleteTask) taskFailed(ctx context.Context, volume *models.SVolume, status string, err error) { + volume.SetStatus(ctx, volumeDeleteTask.UserCred, status, err.Error()) + db.OpsLog.LogEvent(volume, db.ACT_DELETE, err, volumeDeleteTask.UserCred) + logclient.AddActionLogWithStartable(volumeDeleteTask, volume, logclient.ACT_DELETE, err, volumeDeleteTask.UserCred, false) + volumeDeleteTask.SetStageFailed(ctx, jsonutils.NewString(err.Error())) +} + +func (volumeDeleteTask *VolumeDeleteTask) taskComplete(ctx context.Context, volume *models.SVolume) { + volume.RealDelete(ctx, volumeDeleteTask.GetUserCred()) + volumeDeleteTask.SetStageComplete(ctx, nil) +} + +func (volumeDeleteTask *VolumeDeleteTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + volume := obj.(*models.SVolume) + if len(volume.SvrId) == 0 { + volumeDeleteTask.taskComplete(ctx, volume) + return + } + s := auth.GetSession(ctx, volumeDeleteTask.UserCred, "") + _, err := compute.Disks.Delete(s, volume.SvrId, nil) + if err != nil { + if httputils.ErrorCode(err) == 404 { + volumeDeleteTask.taskComplete(ctx, volume) + return + } + volumeDeleteTask.taskFailed(ctx, volume, computeapi.DISK_DEALLOC_FAILED, errors.Wrap(err, "wait status")) + return + } + for i := 0; i < 60; i++ { + _, err := compute.Disks.GetById(s, volume.SvrId, jsonutils.Marshal(map[string]interface{}{ + "scope": "max", + })) + if err != nil { + if strings.Contains(err.Error(), "ResourceNotFoundError") { + volumeDeleteTask.taskComplete(ctx, volume) + return + } + } + time.Sleep(30 * time.Second) + } + volumeDeleteTask.taskFailed(ctx, volume, computeapi.MODELARTS_POOL_STATUS_TIMEOUT, errors.Wrap(err, "wait status")) +} diff --git a/pkg/llm/tasks/worker/doc.go b/pkg/llm/tasks/worker/doc.go new file mode 100644 index 0000000000..d9386a5632 --- /dev/null +++ b/pkg/llm/tasks/worker/doc.go @@ -0,0 +1 @@ +package worker // import "yunion.io/x/onecloud/pkg/llm/tasks/worker" diff --git a/pkg/llm/tasks/worker/startworker.go b/pkg/llm/tasks/worker/startworker.go new file mode 100644 index 0000000000..11ebbae697 --- /dev/null +++ b/pkg/llm/tasks/worker/startworker.go @@ -0,0 +1,38 @@ +package worker + +import ( + "sync" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/appsrv" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/llm/options" +) + +var ( + localTaskWorkerManLock *sync.Mutex + + startTaskWorkerMan *appsrv.SWorkerManager +) + +func init() { + localTaskWorkerManLock = &sync.Mutex{} +} + +func getStartTaskWorkerMan() *appsrv.SWorkerManager { + localTaskWorkerManLock.Lock() + defer localTaskWorkerManLock.Unlock() + + if startTaskWorkerMan != nil { + return startTaskWorkerMan + } + log.Infof("StartTaskWorkerCount %d", options.Options.StartTaskWorkerCount) + startTaskWorkerMan = appsrv.NewWorkerManager("StartTaskWorkerManager", options.Options.StartTaskWorkerCount, 1024, false) + return startTaskWorkerMan +} + +func StartTaskRun(task taskman.ITask, proc func() (jsonutils.JSONObject, error)) { + taskman.LocalTaskRunWithWorkers(task, proc, getStartTaskWorkerMan()) +} diff --git a/pkg/llm/utils/cloudutil.go b/pkg/llm/utils/cloudutil.go new file mode 100644 index 0000000000..ea13de2efe --- /dev/null +++ b/pkg/llm/utils/cloudutil.go @@ -0,0 +1,173 @@ +package utils + +import ( + "context" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/httputils" + "yunion.io/x/pkg/utils" + + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/mcclient/modulebase" + "yunion.io/x/onecloud/pkg/mcclient/modules/compute" +) + +type ResourceInfo struct { + Id string + Name string + Status string +} + +func NewResourceInfo(id, name, status string) ResourceInfo { + return ResourceInfo{ + Id: id, + Name: name, + Status: status, + } +} + +func GetResource[R any](ctx context.Context, man modulebase.Manager, id string) (*R, error) { + if len(id) == 0 { + return nil, errors.Wrapf(httperrors.ErrInvalidStatus, "id is empty") + } + s := auth.GetAdminSession(ctx, "") + resp, err := man.GetById(s, id, jsonutils.Marshal(map[string]interface{}{ + "scope": "max", + })) + if err != nil { + if httputils.ErrorCode(err) == 404 { + return nil, errors.Wrapf(errors.ErrNotFound, "GetById %s", id) + } + return nil, errors.Wrapf(err, "Get") + } + res := new(R) + if err := resp.Unmarshal(res); err != nil { + return nil, errors.Wrap(err, "Unmarshal") + } + return res, nil +} + +func PerformResourceAction[R any](ctx context.Context, man modulebase.Manager, id string, action string) (*R, error) { + s := auth.GetAdminSession(ctx, "") + resp, err := man.PerformAction(s, id, action, nil) + if err != nil { + return nil, errors.Wrapf(err, "PerformAction %s %s %s", man.GetKeyword(), id, action) + } + res := new(R) + if err := resp.Unmarshal(res); err != nil { + return nil, errors.Wrap(err, "Unmarshal") + } + return res, nil +} + +func StopResource[R any](ctx context.Context, man modulebase.Manager, id string) (*R, error) { + return PerformResourceAction[R](ctx, man, id, "stop") +} + +func StartResource[R any](ctx context.Context, man modulebase.Manager, id string) (*R, error) { + return PerformResourceAction[R](ctx, man, id, "start") +} + +func WaitResourceStatus[R any]( + ctx context.Context, + man modulebase.Manager, + id string, + getResInfo func(*R) ResourceInfo, + targetStatus []string, + timeoutSecs int, + intervalSecs int) (*R, error) { + expire := time.Now().Add(time.Second * time.Duration(timeoutSecs)) + for time.Now().Before(expire) { + res, err := GetResource[R](ctx, man, id) + if err != nil { + return nil, errors.Wrapf(err, "GetResource") + } + resInfo := getResInfo(res) + log.Debugf("Wait %s status %#v target status %v", man.GetKeyword(), resInfo.Status, targetStatus) + if utils.IsInStringArray(resInfo.Status, targetStatus) { + return res, nil + } + if strings.Contains(resInfo.Status, "fail") { + return nil, errors.Wrapf(errors.ErrInvalidStatus, "resource %s status %s", resInfo.Name, resInfo.Status) + } + time.Sleep(time.Second * time.Duration(intervalSecs)) + } + return nil, errors.Wrapf(httperrors.ErrTimeout, "wait %s status %s timeout", man.GetKeyword(), targetStatus) +} + +func GetContainer(ctx context.Context, id string) (*computeapi.SContainer, error) { + return GetResource[computeapi.SContainer](ctx, &compute.Containers, id) +} + +func GetServer(ctx context.Context, id string) (*computeapi.ServerDetails, error) { + return GetResource[computeapi.ServerDetails](ctx, &compute.Servers, id) +} + +func WaitContainerStatus(ctx context.Context, id string, targetStatus []string, timeoutSecs int) (*computeapi.SContainer, error) { + return WaitResourceStatus(ctx, &compute.Containers, id, func(ctr *computeapi.SContainer) ResourceInfo { + return NewResourceInfo(ctr.Id, ctr.Name, ctr.Status) + }, targetStatus, timeoutSecs, 1) +} + +func WaitServerStatus(ctx context.Context, id string, targetStatus []string, timeoutSecs int) (*computeapi.ServerDetails, error) { + return WaitResourceStatus(ctx, &compute.Servers, id, func(s *computeapi.ServerDetails) ResourceInfo { + return NewResourceInfo(s.Id, s.Name, s.Status) + }, targetStatus, timeoutSecs, 2) +} + +func WaitDelete[R any](ctx context.Context, man modulebase.Manager, id string, timeoutSecs int) error { + expire := time.Now().Add(time.Second * time.Duration(timeoutSecs)) + for time.Now().Before(expire) { + _, err := GetResource[R](ctx, man, id) + if err != nil { + if errors.Cause(err) == errors.ErrNotFound { + return nil + } + return errors.Wrapf(err, "Get %s %s", man.GetKeyword(), id) + } + time.Sleep(2 * time.Second) + } + return errors.Wrapf(httperrors.ErrTimeout, "wait %s %s deleted timeout", man.GetKeyword(), id) +} + +func UpdateContainer(ctx context.Context, id string, getSpec func(*computeapi.SContainer) *computeapi.ContainerSpec) (*computeapi.SContainer, error) { + s := auth.GetAdminSession(ctx, "") + ctr, err := GetContainer(ctx, id) + if err != nil { + return nil, errors.Wrapf(err, "GetContainer %s", id) + } + curSpecStr := jsonutils.Marshal(ctr.Spec).String() + newSpec := getSpec(ctr) + newSpecStr := jsonutils.Marshal(newSpec).String() + if curSpecStr != newSpecStr { + ctr.Spec = newSpec + resp, err := compute.Containers.Update(s, id, jsonutils.Marshal(ctr)) + if err != nil { + return nil, errors.Wrapf(err, "UpdateContainer %s", id) + } + respCtr := new(computeapi.SContainer) + if err := resp.Unmarshal(respCtr); err != nil { + return nil, errors.Wrapf(err, "Unmarshal") + } + return respCtr, nil + } else { + log.Debugf("container spec not changed, skip update container %s spec", ctr.Name) + } + return ctr, nil +} + +func ExecSyncContainer(ctx context.Context, containerId string, input *computeapi.ContainerExecSyncInput) (jsonutils.JSONObject, error) { + session := auth.GetAdminSession(ctx, "") + output, err := compute.Containers.PerformAction(session, containerId, "exec-sync", jsonutils.Marshal(input)) + if err != nil { + return nil, errors.Wrap(err, "ExecSync") + } + + return output, nil +} diff --git a/pkg/llm/utils/doc.go b/pkg/llm/utils/doc.go new file mode 100644 index 0000000000..a29a8529b7 --- /dev/null +++ b/pkg/llm/utils/doc.go @@ -0,0 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils // import "yunion.io/x/onecloud/pkg/llm/utils" diff --git a/pkg/mcclient/modules/llm/doc.go b/pkg/mcclient/modules/llm/doc.go new file mode 100644 index 0000000000..b69a43a8e2 --- /dev/null +++ b/pkg/mcclient/modules/llm/doc.go @@ -0,0 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package llm // import "yunion.io/x/onecloud/pkg/mcclient/modules/llm" diff --git a/pkg/mcclient/modules/llm/mod_dify.go b/pkg/mcclient/modules/llm/mod_dify.go new file mode 100644 index 0000000000..8d9f5cd62f --- /dev/null +++ b/pkg/mcclient/modules/llm/mod_dify.go @@ -0,0 +1,23 @@ +package llm + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modulebase" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +var ( + Difies DifyManager +) + +func init() { + Difies = DifyManager{ + modules.NewLLMManager("dify", "difies", + []string{"ID", "Name", "Guest_ID", "Containers", "Status"}, + []string{}), + } + modules.Register(&Difies) +} + +type DifyManager struct { + modulebase.ResourceManager +} diff --git a/pkg/mcclient/modules/llm/mod_dify_model.go b/pkg/mcclient/modules/llm/mod_dify_model.go new file mode 100644 index 0000000000..532e6b8bcb --- /dev/null +++ b/pkg/mcclient/modules/llm/mod_dify_model.go @@ -0,0 +1,24 @@ +package llm + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modulebase" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +type DifyModelManager struct { + modulebase.ResourceManager +} + +var ( + DifyModel DifyModelManager +) + +func init() { + DifyModel = DifyModelManager{ + ResourceManager: modules.NewLLMManager("dify_model", "dify_models", + []string{}, + []string{}, + ), + } + modules.Register(&DifyModel) +} diff --git a/pkg/mcclient/modules/llm/mod_image.go b/pkg/mcclient/modules/llm/mod_image.go new file mode 100644 index 0000000000..e4ac45a899 --- /dev/null +++ b/pkg/mcclient/modules/llm/mod_image.go @@ -0,0 +1,24 @@ +package llm + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modulebase" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +type LLMImageManager struct { + modulebase.ResourceManager +} + +var ( + LLMImage LLMImageManager +) + +func init() { + LLMImage = LLMImageManager{ + ResourceManager: modules.NewLLMManager("llm_image", "llm_images", + []string{}, + []string{}, + ), + } + modules.Register(&LLMImage) +} diff --git a/pkg/mcclient/modules/llm/mod_llm.go b/pkg/mcclient/modules/llm/mod_llm.go new file mode 100644 index 0000000000..9ef82d791a --- /dev/null +++ b/pkg/mcclient/modules/llm/mod_llm.go @@ -0,0 +1,23 @@ +package llm + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modulebase" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +var ( + LLMs LLMManager +) + +func init() { + LLMs = LLMManager{ + modules.NewLLMManager("llm", "llms", + []string{}, + []string{}), + } + modules.Register(&LLMs) +} + +type LLMManager struct { + modulebase.ResourceManager +} diff --git a/pkg/mcclient/modules/llm/mod_llm_model.go b/pkg/mcclient/modules/llm/mod_llm_model.go new file mode 100644 index 0000000000..3f88b1c84c --- /dev/null +++ b/pkg/mcclient/modules/llm/mod_llm_model.go @@ -0,0 +1,24 @@ +package llm + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modulebase" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +type LLMModelManager struct { + modulebase.ResourceManager +} + +var ( + LLMModel LLMModelManager +) + +func init() { + LLMModel = LLMModelManager{ + ResourceManager: modules.NewLLMManager("llm_model", "llm_models", + []string{}, + []string{}, + ), + } + modules.Register(&LLMModel) +} diff --git a/pkg/mcclient/modules/llm/mod_task.go b/pkg/mcclient/modules/llm/mod_task.go new file mode 100644 index 0000000000..488a1ebdd6 --- /dev/null +++ b/pkg/mcclient/modules/llm/mod_task.go @@ -0,0 +1,27 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package llm + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modules" + "yunion.io/x/onecloud/pkg/mcclient/modules/tasks" +) + +var LLMTasks tasks.TasksManager +var ArchivedLLMTasks tasks.TasksManager + +func init() { + LLMTasks, ArchivedLLMTasks = tasks.NewTaskManagers(modules.NewLLMManager) +} diff --git a/pkg/mcclient/modules/managers.go b/pkg/mcclient/modules/managers.go index dad54f0dae..160c79b4c7 100644 --- a/pkg/mcclient/modules/managers.go +++ b/pkg/mcclient/modules/managers.go @@ -180,3 +180,9 @@ func NewAPIMapManager(keyword, keywordPlural string, columns, adminColumns []str BaseManager: *modulebase.NewBaseManager(apis.SERVICE_TYPE_APIMAP, "", "", columns, adminColumns), Keyword: keyword, KeywordPlural: keywordPlural} } + +func NewLLMManager(keyword, keywordPlural string, columns, adminColumns []string) modulebase.ResourceManager { + return modulebase.ResourceManager{ + BaseManager: *modulebase.NewBaseManager(apis.SERVICE_TYPE_LLM, "", "", columns, adminColumns), + Keyword: keyword, KeywordPlural: keywordPlural} +} diff --git a/pkg/mcclient/options/llm/dify.go b/pkg/mcclient/options/llm/dify.go new file mode 100644 index 0000000000..a36d1a2a8a --- /dev/null +++ b/pkg/mcclient/options/llm/dify.go @@ -0,0 +1,74 @@ +package llm + +import ( + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/mcclient/options" +) + +type DifyListOptions struct { + LLMBaseListOptions + + DifyModel string `help:"filter by dify model"` +} + +func (o *DifyListOptions) Params() (jsonutils.JSONObject, error) { + params, err := options.ListStructToParams(o) + if err != nil { + return nil, err + } + if o.Used != nil { + params.Set("unused", jsonutils.JSONFalse) + } + return params, nil +} + +type DifyShowOptions struct { + options.BaseShowOptions +} + +func (o *DifyShowOptions) Params() (jsonutils.JSONObject, error) { + return options.StructToParams(o) +} + +type DifyCreateOptions struct { + LLMBaseCreateOptions + + DIFY_MODEL_ID string `help:"dify model id or name" json:"dify_model_id"` +} + +func (o *DifyCreateOptions) Params() (jsonutils.JSONObject, error) { + return jsonutils.Marshal(o), nil +} + +func (o *DifyCreateOptions) GetCountParam() int { + return o.Count +} + +type DifyDeleteOptions struct { + options.BaseIdOptions +} + +func (o *DifyDeleteOptions) GetId() string { + return o.ID +} + +func (o *DifyDeleteOptions) Params() (jsonutils.JSONObject, error) { + return options.StructToParams(o) +} + +type DifyStartOptions struct { + options.BaseIdsOptions +} + +func (o *DifyStartOptions) Params() (jsonutils.JSONObject, error) { + return jsonutils.Marshal(o), nil +} + +type DifyStopOptions struct { + options.BaseIdsOptions +} + +func (o *DifyStopOptions) Params() (jsonutils.JSONObject, error) { + return jsonutils.Marshal(o), nil +} diff --git a/pkg/mcclient/options/llm/dify_model.go b/pkg/mcclient/options/llm/dify_model.go new file mode 100644 index 0000000000..0c060e8b5e --- /dev/null +++ b/pkg/mcclient/options/llm/dify_model.go @@ -0,0 +1,78 @@ +package llm + +import ( + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/mcclient/options" +) + +type DifyModelListOptions struct { + options.BaseListOptions +} + +func (o *DifyModelListOptions) Params() (jsonutils.JSONObject, error) { + return options.ListStructToParams(o) +} + +type DifyModelShowOptions struct { + options.BaseShowOptions +} + +func (o *DifyModelShowOptions) Params() (jsonutils.JSONObject, error) { + return options.StructToParams(o) +} + +type DifyModelCreateOptions struct { + LLMModelBaseCreateOptions + + POSTGRES_IMAGE_ID string `json:"postgres_image_id"` + REDIS_IMAGE_ID string `json:"redis_image_id"` + NGINX_IMAGE_ID string `json:"nginx_image_id"` + DIFY_API_IMAGE_ID string `json:"dify_api_image_id"` + DIFY_PLUGIN_IMAGE_ID string `json:"dify_plugin_image_id"` + DIFY_WEB_IMAGE_ID string `json:"dify_web_image_id"` + DIFY_SANDBOX_IMAGE_ID string `json:"dify_sandbox_image_id"` + DIFY_SSRF_IMAGE_ID string `json:"dify_ssrf_image_id"` + DIFY_WEAVIATE_IMAGE_ID string `json:"dify_weaviate_image_id"` +} + +func (o *DifyModelCreateOptions) Params() (jsonutils.JSONObject, error) { + dict := jsonutils.NewDict() + obj := jsonutils.Marshal(o) + obj.Unmarshal(dict) + + o.LLMModelBaseCreateOptions.Params(dict) + return dict, nil +} + +type DifyModelDeleteOptions struct { + options.BaseIdOptions +} + +func (o *DifyModelDeleteOptions) GetId() string { + return o.ID +} + +func (o *DifyModelDeleteOptions) Params() (jsonutils.JSONObject, error) { + return options.StructToParams(o) +} + +type DifyModelUpdateOptions struct { + LLMModelBaseUpdateOptions + + LlmImageId string + LlmModelName string +} + +func (o *DifyModelUpdateOptions) GetId() string { + return o.ID +} + +func (o *DifyModelUpdateOptions) Params() (jsonutils.JSONObject, error) { + dict := jsonutils.NewDict() + obj := jsonutils.Marshal(o) + obj.Unmarshal(dict) + + o.LLMModelBaseUpdateOptions.Params(dict) + return dict, nil +} diff --git a/pkg/mcclient/options/llm/doc.go b/pkg/mcclient/options/llm/doc.go new file mode 100644 index 0000000000..036ded7960 --- /dev/null +++ b/pkg/mcclient/options/llm/doc.go @@ -0,0 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package llm // import "yunion.io/x/onecloud/pkg/mcclient/options/llm" diff --git a/pkg/mcclient/options/llm/image.go b/pkg/mcclient/options/llm/image.go new file mode 100644 index 0000000000..b49b31a6df --- /dev/null +++ b/pkg/mcclient/options/llm/image.go @@ -0,0 +1,60 @@ +package llm + +import ( + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/apis" + "yunion.io/x/onecloud/pkg/mcclient/options" +) + +type LLMImageShowOptions struct { + options.BaseShowOptions +} + +func (o *LLMImageShowOptions) Params() (jsonutils.JSONObject, error) { + return options.ListStructToParams(o) +} + +type LLMImageListOptions struct { + options.BaseListOptions +} + +func (o *LLMImageListOptions) Params() (jsonutils.JSONObject, error) { + return options.ListStructToParams(o) +} + +type LLMImageCreateOptions struct { + apis.SharableVirtualResourceCreateInput + IMAGE_NAME string + IMAGE_LABEL string + CredentialId string +} + +func (o *LLMImageCreateOptions) Params() (jsonutils.JSONObject, error) { + return jsonutils.Marshal(o), nil +} + +type LLMImageUpdateOptions struct { + apis.SharableVirtualResourceCreateInput + + ID string + IMAGE_NAME string + IMAGE_LABEL string + CredentialId string +} + +func (o *LLMImageUpdateOptions) GetId() string { + return o.ID +} + +func (o *LLMImageUpdateOptions) Params() (jsonutils.JSONObject, error) { + return jsonutils.Marshal(o), nil +} + +type LLMImageDeleteOptions struct { + options.BaseIdOptions +} + +func (o *LLMImageDeleteOptions) GetId() string { + return o.ID +} diff --git a/pkg/mcclient/options/llm/llm.go b/pkg/mcclient/options/llm/llm.go new file mode 100644 index 0000000000..26de89572b --- /dev/null +++ b/pkg/mcclient/options/llm/llm.go @@ -0,0 +1,99 @@ +package llm + +import ( + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/mcclient/options" +) + +type LLMBaseListOptions struct { + options.BaseListOptions + Host string `help:"filter by host"` + LLMStatus []string `help:"filter by server status"` + + ListenPort int `help:"filter by listen port"` + PublicIp string `help:"filter by public ip"` + VolumeId string `help:"filter by volume id"` + Unused *bool `help:"filter by unused"` + Used *bool `help:"filter by used"` +} + +type LLMListOptions struct { + LLMBaseListOptions + + LlmModel string `help:"filter by llm model"` + LlmImage string `help:"filter by llm image"` +} + +func (o *LLMListOptions) Params() (jsonutils.JSONObject, error) { + params, err := options.ListStructToParams(o) + if err != nil { + return nil, err + } + if o.Used != nil { + params.Set("unused", jsonutils.JSONFalse) + } + return params, nil +} + +type LLMShowOptions struct { + options.BaseShowOptions +} + +func (o *LLMShowOptions) Params() (jsonutils.JSONObject, error) { + return options.StructToParams(o) +} + +type LLMBaseCreateOptions struct { + options.BaseCreateOptions + + AutoStart bool + ProjectId string + PreferHost string + + BandwidthMb int + + Count int `default:"1" help:"batch create count" json:"-"` +} + +type LLMCreateOptions struct { + LLMBaseCreateOptions + + LLM_MODEL_ID string `help:"llm model id or name" json:"llm_model_id"` +} + +func (o *LLMCreateOptions) Params() (jsonutils.JSONObject, error) { + return jsonutils.Marshal(o), nil +} + +func (o *LLMCreateOptions) GetCountParam() int { + return o.Count +} + +type LLMDeleteOptions struct { + options.BaseIdOptions +} + +func (o *LLMDeleteOptions) GetId() string { + return o.ID +} + +func (o *LLMDeleteOptions) Params() (jsonutils.JSONObject, error) { + return options.StructToParams(o) +} + +type LLMStartOptions struct { + options.BaseIdsOptions +} + +func (o *LLMStartOptions) Params() (jsonutils.JSONObject, error) { + return jsonutils.Marshal(o), nil +} + +type LLMStopOptions struct { + options.BaseIdsOptions +} + +func (o *LLMStopOptions) Params() (jsonutils.JSONObject, error) { + return jsonutils.Marshal(o), nil +} diff --git a/pkg/mcclient/options/llm/llm_model.go b/pkg/mcclient/options/llm/llm_model.go new file mode 100644 index 0000000000..06cfd96084 --- /dev/null +++ b/pkg/mcclient/options/llm/llm_model.go @@ -0,0 +1,74 @@ +package llm + +import ( + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/mcclient/options" +) + +type LLMModelListOptions struct { + options.BaseListOptions + + LLMType string `json:"llm_type" choices:"ollama"` +} + +func (o *LLMModelListOptions) Params() (jsonutils.JSONObject, error) { + return options.ListStructToParams(o) +} + +type LLMModelShowOptions struct { + options.BaseShowOptions +} + +func (o *LLMModelShowOptions) Params() (jsonutils.JSONObject, error) { + return options.StructToParams(o) +} + +type LLMModelCreateOptions struct { + LLMModelBaseCreateOptions + + LLM_IMAGE_ID string `json:"llm_image_id"` + LLM_TYPE string `json:"llm_type" choices:"ollama"` + LLM_MODEL_NAME string `help:"specific model of large language model, for example: qwen3:32b" json:"llm_model_name"` +} + +func (o *LLMModelCreateOptions) Params() (jsonutils.JSONObject, error) { + dict := jsonutils.NewDict() + obj := jsonutils.Marshal(o) + obj.Unmarshal(dict) + + o.LLMModelBaseCreateOptions.Params(dict) + return dict, nil +} + +type LLMModelDeleteOptions struct { + options.BaseIdOptions +} + +func (o *LLMModelDeleteOptions) GetId() string { + return o.ID +} + +func (o *LLMModelDeleteOptions) Params() (jsonutils.JSONObject, error) { + return options.StructToParams(o) +} + +type LLMModelUpdateOptions struct { + LLMModelBaseUpdateOptions + + LlmImageId string + LlmModelName string +} + +func (o *LLMModelUpdateOptions) GetId() string { + return o.ID +} + +func (o *LLMModelUpdateOptions) Params() (jsonutils.JSONObject, error) { + dict := jsonutils.NewDict() + obj := jsonutils.Marshal(o) + obj.Unmarshal(dict) + + o.LLMModelBaseUpdateOptions.Params(dict) + return dict, nil +} diff --git a/pkg/mcclient/options/llm/llm_model_base.go b/pkg/mcclient/options/llm/llm_model_base.go new file mode 100644 index 0000000000..93b675e578 --- /dev/null +++ b/pkg/mcclient/options/llm/llm_model_base.go @@ -0,0 +1,220 @@ +package llm + +import ( + "fmt" + "strconv" + "strings" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/apis" + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + api "yunion.io/x/onecloud/pkg/apis/llm" +) + +type LLMModelBaseCreateOptions struct { + apis.SharableVirtualResourceCreateInput + + CPU int + MEMORY int `help:"memory size MB"` + DISK_SIZE int `help:"disk size MB"` + + Bandwidth int + StorageType string + // DiskOverlay string `help:"disk overlay, e.g. /opt/steam-data/base:/opt/steam-data/games"` + TemplateId string + PortMappings []string `help:"port mapping in the format of protocol:port[:prefix][:first_port_offset][:env_key=env_value], e.g. tcp:5555:192.168.0.0/16:5:WOLF_BASE_PORT=20000"` + Devices []string `help:"device info in the format of model[:path[:dev_type]], e.g. 'GeForce RTX 4060'"` + + Env []string `help:"env in format of key=value"` + Property []string `help:"extra properties of key=value, e.g. tango32=true"` + + // MountedApps []string `help:"mounted apps, e.g. com.tencent.tmgp.sgame/1.0.0"` + + Entrypoint string `help:"entrypoint"` +} + +func (o *LLMModelBaseCreateOptions) Params(dict *jsonutils.JSONDict) error { + vol := api.Volume{ + SizeMB: o.DISK_SIZE, + TemplateId: o.TemplateId, + StorageType: o.StorageType, + } + vols := []api.Volume{vol} + dict.Set("volumes", jsonutils.Marshal(vols)) + fetchPortmappings(o.PortMappings, dict) + fetchDevices(o.Devices, dict) + fetchEnvs(o.Env, dict) + fetchProperties(o.Property, dict) + + return nil +} + +type LLMModelBaseUpdateOptions struct { + apis.SharableVirtualResourceBaseUpdateInput + + ID string + + Cpu *int + Memory *int `help:"memory size GB"` + DiskSize *int `help:"disk size MB"` + StorageType string + TemplateId string + NoTemplate bool `json:"-" help:"remove template"` + Bandwidth *int + // Dpi *int + // Fps *int + PortMappings []string `help:"port mapping in the format of protocol:port[:prefix][:first_port_offset], e.g. tcp:5555:192.168.0.0/16,10.10.0.0/16:1000"` + Devices []string `help:"device info in the format of model[:path[:dev_type]], e.g. QuadraT2A:/dev/nvme1n1, Device::VASTAITECH_GPU"` + Env []string `help:"env in the format of key=value, e.g. AUTHENTICATION_PATH=/bupt-test/"` + Property []string `help:"extra properties of key=value, e.g. tango32=true"` + + // MountedApps []string `help:"mounted apps, e.g. com.tencent.tmgp.sgame/1.0.0"` + + SyncImage bool `help:"request sync image" json:"-"` + ClearSyncImage bool `help:"request clear sync image flag" json:"-"` + + Entrypoint string `help:"entrypoint"` +} + +func (o *LLMModelBaseUpdateOptions) Params(dict *jsonutils.JSONDict) error { + if o.NoTemplate { + dict.Set("template_id", jsonutils.NewString("")) + } + if o.SyncImage { + dict.Set("request_sync_image", jsonutils.JSONTrue) + } else if o.ClearSyncImage { + dict.Set("request_sync_image", jsonutils.JSONFalse) + } + if o.DiskSize != nil && *o.DiskSize > 0 { + dict.Set("disk_size_mb", jsonutils.NewInt(int64(*o.DiskSize))) + } + + fetchPortmappings(o.PortMappings, dict) + fetchDevices(o.Devices, dict) + fetchEnvs(o.Env, dict) + fetchProperties(o.Property, dict) + // fetchMountedApps(o.MountedApps, dict) + return nil +} + +func fetchPortmappings(pmStrs []string, dict *jsonutils.JSONDict) { + pms := make([]api.PortMapping, 0) + for _, pm := range pmStrs { + segs := strings.Split(pm, ":") + if len(segs) > 1 { + port, _ := strconv.ParseInt(segs[1], 10, 64) + var remoteIps []string + if len(segs) > 2 { + for _, ip := range strings.Split(segs[2], ",") { + ip = strings.TrimSpace(ip) + if len(ip) > 0 { + remoteIps = append(remoteIps, ip) + } + } + } + firstPortOffset := 0 + var err error + if len(segs) > 3 { + firstPortOffset, err = strconv.Atoi(segs[3]) + if err != nil { + panic(fmt.Sprintf("parse firstPortOffset: %s", err)) + } + } + pm := api.PortMapping{ + Protocol: segs[0], + ContainerPort: int(port), + RemoteIps: remoteIps, + } + if firstPortOffset >= 0 { + pm.FirstPortOffset = &firstPortOffset + } + if len(segs) > 4 { + envs := make([]computeapi.GuestPortMappingEnv, 0) + for _, env := range strings.Split(segs[4], ",") { + parts := strings.Split(env, "=") + if len(parts) != 2 { + panic(fmt.Sprintf("parse env: %s", env)) + } + key := parts[0] + valType := parts[1] + switch valType { + case "port": + envs = append(envs, computeapi.GuestPortMappingEnv{Key: key, ValueFrom: computeapi.GuestPortMappingEnvValueFromPort}) + case "host_port": + envs = append(envs, computeapi.GuestPortMappingEnv{Key: key, ValueFrom: computeapi.GuestPortMappingEnvValueFromHostPort}) + default: + panic(fmt.Sprintf("wrong env type: %q", valType)) + } + } + pm.Envs = envs + } + + pms = append(pms, pm) + } + } + if len(pms) > 0 { + dict.Set("port_mappings", jsonutils.Marshal(pms)) + } +} + +func fetchDevices(devStrs []string, dict *jsonutils.JSONDict) { + devs := make([]api.Device, 0) + for _, dev := range devStrs { + segs := strings.Split(dev, ":") + if len(segs) > 0 { + devpath := "" + devType := "" + if len(segs) > 1 { + devpath = segs[1] + } + if len(segs) > 2 { + devType = segs[2] + } + devs = append(devs, api.Device{ + Model: segs[0], + DevicePath: devpath, + DevType: devType, + }) + } + } + if len(devs) > 0 { + dict.Set("devices", jsonutils.Marshal(devs)) + } +} + +func fetchEnvs(EnvStrs []string, dict *jsonutils.JSONDict) { + envs := make(api.Envs, 0) + for _, env := range EnvStrs { + pos := strings.Index(env, "=") + if pos > 0 { + key := strings.TrimSpace(env[:pos]) + val := strings.TrimSpace(env[pos+1:]) + envs = append(envs, api.Env{Key: key, Value: val}) + } + } + if len(envs) > 0 { + dict.Set("envs", jsonutils.Marshal(envs)) + } +} + +func fetchProperties(propStrs []string, dict *jsonutils.JSONDict) { + props := make(map[string]string, 0) + for _, env := range propStrs { + pos := strings.Index(env, "=") + if pos > 0 { + key := strings.TrimSpace(env[:pos]) + val := strings.TrimSpace(env[pos+1:]) + props[key] = val + } + } + if len(props) > 0 { + dict.Set("properties", jsonutils.Marshal(props)) + } +} + +// func fetchMountedApps(apps []string, dict *jsonutils.JSONDict) { +// if len(apps) > 0 { +// dict.Set("mounted_apps", jsonutils.Marshal(apps)) +// } +// } diff --git a/scripts/sync_dify_images.sh b/scripts/sync_dify_images.sh new file mode 100644 index 0000000000..3d6b7e25d4 --- /dev/null +++ b/scripts/sync_dify_images.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# 用法: +# ./sync-images.sh +# 示例: +# ./sync-images.sh crpi-nf3abu98o8qf9y2x.cn-beijing.personal.cr.aliyuncs.com/eikoh + +set -euo pipefail + +if [ $# -ne 1 ]; then + echo "用法: $0 " + echo "例如: $0 crpi-nf3abu98o8qf9y2x.cn-beijing.personal.cr.aliyuncs.com/eikoh" + exit 1 +fi + +TARGET_REGISTRY="$1" +SOURCE_REGISTRY="docker.io" + +# ---------------------------- +# 要同步的镜像列表 +# ---------------------------- +IMAGES=( + "nginx:latest" + "redis:6-alpine" + "postgres:15-alpine" + "langgenius/dify-api:1.7.2" + "langgenius/dify-sandbox:0.2.12" + "langgenius/dify-plugin-daemon:0.2.0-local" + "langgenius/dify-web:1.7.2" + "ubuntu/squid:latest" + "semitechnologies/weaviate:1.19.0" +) + +for image in "${IMAGES[@]}"; do + # 拆分 name 和 tag + if [[ "$image" == *":"* ]]; then + name="${image%%:*}" # 冒号前 + tag="${image##*:}" # 冒号后 + else + name="$image" + tag="latest" + fi + + short_name="${name##*/}" # 目标镜像只取最后一级名字 + + SRC="docker://${SOURCE_REGISTRY}/${name}:${tag}" + DST="docker://${TARGET_REGISTRY}/${short_name}:${tag}" + + echo + echo "Sync dify image" + echo " Source: ${SRC}" + echo " Target: ${DST}" + echo + + skopeo copy "${SRC}" "${DST}" + + echo "Completed: ${short_name}:${tag}" +done + +echo "All images sync completed"