diff --git a/Gopkg.lock b/Gopkg.lock index ba8dc82308..dd0d306013 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -2,22 +2,27 @@ [[projects]] + digest = "1:f7a3877d7116adfcdcd641f9f6ad18fc2126bd5c080d8854cc3b395a01e0b7f3" name = "github.com/DataDog/dd-trace-go" packages = [ "opentracing", "tracer", - "tracer/ext" + "tracer/ext", ] + pruneopts = "UT" revision = "27617015d45e6cd550b9a7ac7715c37cc2f7d020" version = "v0.6.1" [[projects]] + digest = "1:66b8ed452b31eb9075bc53295952487c333a9cb555de57ed61f5664c058d9050" name = "github.com/Shopify/sarama" packages = ["."] + pruneopts = "UT" revision = "35324cf48e33d8260e1c7c18854465a904ade249" version = "v1.17.0" [[projects]] + digest = "1:37a8ee6652ff3c42043fa26c0930858d1a72f26928ae26e5261fe81c5f0c2e8d" name = "github.com/aliyun/alibaba-cloud-sdk-go" packages = [ "sdk", @@ -28,47 +33,61 @@ "sdk/errors", "sdk/requests", "sdk/responses", - "sdk/utils" + "sdk/utils", ] + pruneopts = "UT" revision = "f53c66e9cf53d68bcf063643c09cc667efa0e613" version = "1.23.3" [[projects]] + digest = "1:fff66556e25e676b296f9f2c78853831911e6ae41d84303a9be7a534bd1c0901" name = "github.com/aliyun/aliyun-oss-go-sdk" packages = ["oss"] + pruneopts = "UT" revision = "36bf7aa2f916f274137aedd05d1d36f6a3c0f4a6" version = "1.9.0" [[projects]] + digest = "1:8f5416c7f59da8600725ae1ff00a99af1da8b04c211ae6f3c8f8bcab0164f650" name = "github.com/aokoli/goutils" packages = ["."] + pruneopts = "UT" revision = "3391d3790d23d03408670993e957e8f408993c34" version = "v1.0.1" [[projects]] + digest = "1:eee9386329f4fcdf8d6c0def0c9771b634bdd5ba460d888aa98c17d59b37a76c" name = "github.com/apache/thrift" packages = ["lib/go/thrift"] + pruneopts = "UT" revision = "e59b73d3c2bf1c328ccb78e683c0462fa1a473c7" [[projects]] branch = "master" + digest = "1:d6afaeed1502aa28e80a4ed0981d570ad91b2579193404256ce672ed0a609e0d" name = "github.com/beorn7/perks" packages = ["quantile"] + pruneopts = "UT" revision = "3a771d992973f24aa725d07868b467d1ddfceafb" [[projects]] + digest = "1:b520b55fc1146c5b0eea03b07233f7a3d4a9be985c037c91ea6b82ecb81bd521" name = "github.com/bitly/go-simplejson" packages = ["."] + pruneopts = "UT" revision = "aabad6e819789e569bd6aabf444c935aa9ba1e44" version = "v0.5.0" [[projects]] + digest = "1:a2486f94bda33b301614ff5cfce102b7eed4c923fa650c2b83dba1fcf2117466" name = "github.com/c-bata/go-prompt" packages = ["."] + pruneopts = "UT" revision = "c52492ff1b386e5c0ba5271b5eaad165fab09eca" version = "v0.2.2" [[projects]] + digest = "1:40098afbdd06a76dee4b6bcb85a22fed81ac9d2ebaf775c91e558c80f57aaff1" name = "github.com/coredns/coredns" packages = [ "core/dnsserver", @@ -118,108 +137,140 @@ "plugin/reload", "plugin/test", "plugin/trace", - "request" + "request", ] + pruneopts = "UT" revision = "2e322f6e8a54f18c6aef9c25a7c432c291a3d9f7" version = "v1.2.0" [[projects]] + digest = "1:a2c1d0e43bd3baaa071d1b9ed72c27d78169b2b269f71c105ac4ba34b1be4a39" name = "github.com/davecgh/go-spew" packages = ["spew"] + pruneopts = "UT" revision = "346938d642f2ec3594ed81d874461961cd0faa76" version = "v1.1.0" [[projects]] branch = "master" + digest = "1:737260890e4b00d73a76993eac1a772cf6a57292f723c47e26f005821efcabcf" name = "github.com/dnstap/golang-dnstap" packages = ["."] + pruneopts = "UT" revision = "2cf77a2b5e11ac8d0ba3892772ac8e1f7b528344" [[projects]] + digest = "1:1f0c7ab489b407a7f8f9ad16c25a504d28ab461517a971d341388a56156c1bd7" name = "github.com/eapache/go-resiliency" packages = ["breaker"] + pruneopts = "UT" revision = "ea41b0fad31007accc7f806884dcdf3da98b79ce" version = "v1.1.0" [[projects]] branch = "master" + digest = "1:0448d1c1a596941c608762912fe7c865f88d9ffa45afb8af1edcf1401762bf7e" name = "github.com/eapache/go-xerial-snappy" packages = ["."] + pruneopts = "UT" revision = "040cc1a32f578808623071247fdbd5cc43f37f5f" [[projects]] + digest = "1:444b82bfe35c83bbcaf84e310fb81a1f9ece03edfed586483c869e2c046aef69" name = "github.com/eapache/queue" packages = ["."] + pruneopts = "UT" revision = "44cc805cf13205b55f69e14bcb69867d1ae92f98" version = "v1.1.0" [[projects]] branch = "master" + digest = "1:a76c5e933af8688d9dcfbe718f2b0bfc955208fe0e47e74218b45f1df83ef2ae" name = "github.com/farsightsec/golang-framestream" packages = ["."] + pruneopts = "UT" revision = "c06a5734334d9629b3db143d74b47eb94ea68612" [[projects]] + digest = "1:865079840386857c809b72ce300be7580cb50d3d3129ce11bf9aa6ca2bc1934a" name = "github.com/fatih/color" packages = ["."] + pruneopts = "UT" revision = "5b77d2a35fb0ede96d138fc9a99f5c9b6aef11b4" version = "v1.7.0" [[projects]] branch = "master" + digest = "1:50a46ab1d5edbbdd55125b4d37f1bf503d0807c26461f9ad7b358d6006641d09" name = "github.com/flynn/go-shlex" packages = ["."] + pruneopts = "UT" revision = "3f9db97f856818214da2e1057f8ad84803971cff" [[projects]] + digest = "1:2cd7915ab26ede7d95b8749e6b1f933f1c6d5398030684e6505940a10f31cfda" name = "github.com/ghodss/yaml" packages = ["."] + pruneopts = "UT" revision = "0ca9ea5df5451ffdf184b4428c902747c2c11cd7" version = "v1.0.0" [[projects]] branch = "master" + digest = "1:36fe9527deed01d2a317617e59304eb2c4ce9f8a24115bcc5c2e37b3aee5bae4" name = "github.com/gin-contrib/sse" packages = ["."] + pruneopts = "UT" revision = "22d885f9ecc78bf4ee5d72b937e4bbcdc58e8cae" [[projects]] + digest = "1:ab00ecdc9607692ffd5fb1ac21a9ba2d6cc95bae011f327022c0b1c323d218e2" name = "github.com/gin-gonic/gin" packages = [ "binding", - "render" + "render", ] + pruneopts = "UT" revision = "d459835d2b077e44f7c9b453505ee29881d5d12d" version = "v1.2" [[projects]] + digest = "1:31a18dae27a29aa074515e43a443abfd2ba6deb6d69309d8d7ce789c45f34659" name = "github.com/go-logfmt/logfmt" packages = ["."] + pruneopts = "UT" revision = "390ab7935ee28ec6b286364bba9b4dd6410cb3d5" version = "v0.3.0" [[projects]] + digest = "1:adea5a94903eb4384abef30f3d878dc9ff6b6b5b0722da25b82e5169216dfb61" name = "github.com/go-sql-driver/mysql" packages = ["."] + pruneopts = "UT" revision = "d523deb1b23d913de5bdada721a6071e71283618" version = "v1.4.0" [[projects]] + digest = "1:34e709f36fd4f868fb00dbaf8a6cab4c1ae685832d392874ba9d7c5dec2429d1" name = "github.com/gogo/protobuf" packages = [ "proto", - "sortkeys" + "sortkeys", ] + pruneopts = "UT" revision = "636bf0302bc95575d69441b25a2603156ffdddf1" version = "v1.1.1" [[projects]] + digest = "1:94d8c9e3a4954505ca89243e2121180972eecf8e3d160f3b34a48825ddd95cc1" name = "github.com/golang-plus/errors" packages = ["."] + pruneopts = "UT" revision = "a61c34ead85085f3e1b87b8c04e49bbf60c99fb1" version = "v1.0.0" [[projects]] + digest = "1:224bf08a201edf15ec9f086549321257d5a2ee0d23dc649e7e6ad90ce5502439" name = "github.com/golang-plus/uuid" packages = [ ".", @@ -229,205 +280,275 @@ "internal/namebased/md5", "internal/namebased/sha1", "internal/random", - "internal/timebased" + "internal/timebased", ] + pruneopts = "UT" revision = "a0510c37ef1f49c8290c3a9911d0f7f9ed24d335" version = "v1.0.0" [[projects]] branch = "master" + digest = "1:1ba1d79f2810270045c328ae5d674321db34e3aae468eb4233883b473c5c0467" name = "github.com/golang/glog" packages = ["."] + pruneopts = "UT" revision = "23def4e6c14b4da8ac2ed8007337bc5eb5007998" [[projects]] + digest = "1:17fe264ee908afc795734e8c4e63db2accabaf57326dbf21763a7d6b86096260" name = "github.com/golang/protobuf" packages = [ "proto", "ptypes", "ptypes/any", "ptypes/duration", - "ptypes/timestamp" + "ptypes/timestamp", ] + pruneopts = "UT" revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265" version = "v1.1.0" [[projects]] branch = "master" + digest = "1:4a0c6bb4805508a6287675fac876be2ac1182539ca8a32468d8128882e9d5009" name = "github.com/golang/snappy" packages = ["."] + pruneopts = "UT" revision = "2e65f85255dbc3072edf28d6b5b8efc472979f5a" [[projects]] branch = "master" + digest = "1:9887333bbef17574b1db5f9893ea137ac44107235d624408a3ac9e0b98fbb2cb" name = "github.com/google/btree" packages = ["."] + pruneopts = "UT" revision = "e89373fe6b4a7413d7acd6da1725b83ef713e6e4" [[projects]] branch = "master" + digest = "1:3ee90c0d94da31b442dde97c99635aaafec68d0b8a3c12ee2075c6bdabeec6bb" name = "github.com/google/gofuzz" packages = ["."] + pruneopts = "UT" revision = "24818f796faf91cd76ec7bddd72458fbced7a6c1" [[projects]] + digest = "1:8f8811f9be822914c3a25c6a071e93beb4c805d7b026cbf298bc577bc1cc945b" name = "github.com/google/uuid" packages = ["."] + pruneopts = "UT" revision = "064e2069ce9c359c118179501254f67d7d37ba24" version = "0.2" [[projects]] + digest = "1:65c4414eeb350c47b8de71110150d0ea8a281835b1f386eacaa3ad7325929c21" name = "github.com/googleapis/gnostic" packages = [ "OpenAPIv2", "compiler", - "extensions" + "extensions", ] + pruneopts = "UT" revision = "7c663266750e7d82587642f65e60bc4083f1f84e" version = "v0.2.0" [[projects]] branch = "master" + digest = "1:4e08dc2383a46b3107f0b34ca338c4459e8fc8ee90e46a60e728aa8a2b21d558" + name = "github.com/gosuri/uitable" + packages = [ + ".", + "util/strutil", + "util/wordwrap", + ] + pruneopts = "UT" + revision = "36ee7e946282a3fb1cfecd476ddc9b35d8847e42" + +[[projects]] + branch = "master" + digest = "1:86c1210529e69d69860f2bb3ee9ccce0b595aa3f9165e7dd1388e5c612915888" name = "github.com/gregjones/httpcache" packages = [ ".", - "diskcache" + "diskcache", ] + pruneopts = "UT" revision = "9cad4c3443a7200dd6400aef47183728de563a38" [[projects]] branch = "master" + digest = "1:1a1206efd03a54d336dce7bb8719e74f2f8932f661cb9f57d5813a1d99c083d8" name = "github.com/grpc-ecosystem/grpc-opentracing" packages = ["go/otgrpc"] + pruneopts = "UT" revision = "8e809c8a86450a29b90dcc9efbf062d0fe6d9746" [[projects]] + digest = "1:8eb1de8112c9924d59bf1d3e5c26f5eaa2bfc2a5fcbb92dc1c2e4546d695f277" name = "github.com/imdario/mergo" packages = ["."] + pruneopts = "UT" revision = "9f23e2d6bd2a77f959b2bf6acdbefd708a83a4a4" version = "v0.3.6" [[projects]] + digest = "1:b6f4477f09f11785097a121811f3a90d8d26cab73c1aa71abda859bdf6ad2334" name = "github.com/jinzhu/gorm" packages = [ ".", - "dialects/mysql" + "dialects/mysql", ] + pruneopts = "UT" revision = "6ed508ec6a4ecb3531899a69cbc746ccf65a4166" version = "v1.9.1" [[projects]] branch = "master" + digest = "1:fd97437fbb6b7dce04132cf06775bd258cce305c44add58eb55ca86c6c325160" name = "github.com/jinzhu/inflection" packages = ["."] + pruneopts = "UT" revision = "04140366298a54a039076d798123ffa108fff46c" [[projects]] + digest = "1:b87714e57a511d88f307aba7d5b63522da12bed0a050889c81272fc50f71100e" name = "github.com/jmespath/go-jmespath" packages = ["."] + pruneopts = "UT" revision = "3433f3ea46d9f8019119e7dd41274e112a2359a9" version = "0.2.2" [[projects]] + digest = "1:eaefc85d32c03e5f0c2b88ea2f79fce3d993e2c78316d21319575dd4ea9153ca" name = "github.com/json-iterator/go" packages = ["."] + pruneopts = "UT" revision = "ab8a2e0c74be9d3be70b3184d9acc634935ded82" version = "1.1.4" [[projects]] branch = "master" + digest = "1:a64e323dc06b73892e5bb5d040ced475c4645d456038333883f58934abbf6f72" name = "github.com/kr/logfmt" packages = ["."] + pruneopts = "UT" revision = "b84e30acd515aadc4b783ad4ff83aff3299bdfe0" [[projects]] + digest = "1:c658e84ad3916da105a761660dcaeb01e63416c8ec7bc62256a9b411a05fcd67" name = "github.com/mattn/go-colorable" packages = ["."] + pruneopts = "UT" revision = "167de6bfdfba052fa6b2d3664c8f5272e23c9072" version = "v0.0.9" [[projects]] + digest = "1:d4d17353dbd05cb52a2a52b7fe1771883b682806f68db442b436294926bbfafb" name = "github.com/mattn/go-isatty" packages = ["."] + pruneopts = "UT" revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39" version = "v0.0.3" [[projects]] branch = "master" + digest = "1:cdb899c199f907ac9fb50495ec71212c95cb5b0e0a8ee0800da0238036091033" name = "github.com/mattn/go-runewidth" packages = ["."] + pruneopts = "UT" revision = "ce7b0b5c7b45a81508558cd1dba6bb1e4ddb51bb" [[projects]] branch = "master" + digest = "1:d775613e6db40f54d34cc8b319ee5c2df56752f9f5e55e2f6004d11684bdc8fd" name = "github.com/mattn/go-tty" packages = ["."] + pruneopts = "UT" revision = "931426f7535ac39720c8909d70ece5a41a2502a6" [[projects]] + digest = "1:ff5ebae34cfbf047d505ee150de27e60570e8c394b3b8fdbb720ff6ac71985fc" name = "github.com/matttproud/golang_protobuf_extensions" packages = ["pbutil"] + pruneopts = "UT" revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c" version = "v1.0.1" [[projects]] branch = "master" + digest = "1:2b32af4d2a529083275afc192d1067d8126b578c7a9613b26600e4df9c735155" name = "github.com/mgutz/ansi" packages = ["."] + pruneopts = "UT" revision = "9520e82c474b0a04dd04f8a40959027271bab992" [[projects]] + digest = "1:2e9ce249cbffeaf99b7b8d13b2191591bda8421f416e1366bdde8f7785f034cd" name = "github.com/mholt/caddy" packages = [ ".", "caddyfile", "onevent/hook", - "startupshutdown" + "startupshutdown", ] + pruneopts = "UT" revision = "d3f338ddab9ee24b376b5c9c51e202581e2d43ba" version = "v0.10.11" [[projects]] + digest = "1:463e4140189f8194f9121ca1c7fe3b8e9e9a2ab3d949b43c835c21034927dc62" name = "github.com/miekg/dns" packages = ["."] + pruneopts = "UT" revision = "5a2b9fab83ff0f8bfc99684bd5f43a37abe560f1" version = "v1.0.8" [[projects]] + digest = "1:33422d238f147d247752996a26574ac48dcf472976eda7f5134015f06bf16563" name = "github.com/modern-go/concurrent" packages = ["."] + pruneopts = "UT" revision = "bacd9c7ef1dd9b15be4a9909b8ac7a4e313eec94" version = "1.0.3" [[projects]] + digest = "1:e32bdbdb7c377a07a9a46378290059822efdce5c8d96fe71940d87cb4f918855" name = "github.com/modern-go/reflect2" packages = ["."] + pruneopts = "UT" revision = "4b7aa43c6742a2c18fdef89dd197aaae7dac7ccd" version = "1.0.1" [[projects]] branch = "master" + digest = "1:7aefb397a53fc437c90f0fdb3e1419c751c5a3a165ced52325d5d797edf1aca6" name = "github.com/moul/http2curl" packages = ["."] + pruneopts = "UT" revision = "9ac6cf4d929b2fa8fd2d2e6dec5bb0feb4f4911d" [[projects]] branch = "master" + digest = "1:2da0e5077ed40453dc281b9a2428d84cf6ad14063aed189f6296ca5dd25cf13d" name = "github.com/opentracing-contrib/go-observer" packages = ["."] + pruneopts = "UT" revision = "a52f2342449246d5bcc273e65cbdcfa5f7d6c63c" [[projects]] + digest = "1:450b7623b185031f3a456801155c8320209f75d0d4c4e633c6b1e59d44d6e392" name = "github.com/opentracing/opentracing-go" packages = [ ".", "ext", - "log" + "log", ] + pruneopts = "UT" revision = "1949ddbfd147afd4d964a9f00b24eb291e0e7c38" version = "v1.0.2" [[projects]] + digest = "1:a427dd5035907bf7ad9a1fa269805f3a9ead9cfab056b32e2d86c230e28e31c0" name = "github.com/openzipkin/zipkin-go-opentracing" packages = [ ".", @@ -435,123 +556,157 @@ "thrift/gen-go/scribe", "thrift/gen-go/zipkincore", "types", - "wire" + "wire", ] + pruneopts = "UT" revision = "26cf9707480e6b90e5eff22cf0bbf05319154232" version = "v0.3.4" [[projects]] branch = "master" + digest = "1:3bf17a6e6eaa6ad24152148a631d18662f7212e21637c2699bff3369b7f00fa2" name = "github.com/petar/GoLLRB" packages = ["llrb"] + pruneopts = "UT" revision = "53be0d36a84c2a886ca057d34b6aa4468df9ccb4" [[projects]] + digest = "1:0e7775ebbcf00d8dd28ac663614af924411c868dca3d5aa762af0fae3808d852" name = "github.com/peterbourgon/diskv" packages = ["."] + pruneopts = "UT" revision = "5f041e8faa004a95c88a202771f4cc3e991971e6" version = "v2.0.1" [[projects]] + digest = "1:29803f52611cbcc1dfe55b456e9fdac362af7248b3d29d7ea1bec0a12e71dff4" name = "github.com/pierrec/lz4" packages = [ ".", - "internal/xxh32" + "internal/xxh32", ] + pruneopts = "UT" revision = "1958fd8fff7f115e79725b1288e0b878b3e06b00" version = "v2.0.3" [[projects]] branch = "master" + digest = "1:4f43b3c1b7e44980a5f3c593f8bf0e18844dc44f451a071c93e77e28cf990db6" name = "github.com/pkg/term" packages = ["termios"] + pruneopts = "UT" revision = "cda20d4ac917ad418d86e151eff439648b06185b" [[projects]] + digest = "1:0028cb19b2e4c3112225cd871870f2d9cf49b9b4276531f03438a88e94be86fe" name = "github.com/pmezard/go-difflib" packages = ["difflib"] + pruneopts = "UT" revision = "792786c7400a136282c1664665ae0a8db921c6c2" version = "v1.0.0" [[projects]] + digest = "1:d14a5f4bfecf017cb780bdde1b6483e5deb87e12c332544d2c430eda58734bcb" name = "github.com/prometheus/client_golang" packages = [ "prometheus", - "prometheus/promhttp" + "prometheus/promhttp", ] + pruneopts = "UT" revision = "c5b7fccd204277076155f10851dad72b76a49317" version = "v0.8.0" [[projects]] branch = "master" + digest = "1:2d5cd61daa5565187e1d96bae64dbbc6080dacf741448e9629c64fd93203b0d4" name = "github.com/prometheus/client_model" packages = ["go"] + pruneopts = "UT" revision = "5c3871d89910bfb32f5fcab2aa4b9ec68e65a99f" [[projects]] branch = "master" + digest = "1:e469cd65badf7694aeb44874518606d93c1d59e7735d3754ad442782437d3cc3" name = "github.com/prometheus/common" packages = [ "expfmt", "internal/bitbucket.org/ww/goautoneg", - "model" + "model", ] + pruneopts = "UT" revision = "7600349dcfe1abd18d72d3a1770870d9800a7801" [[projects]] branch = "master" + digest = "1:8c49953a1414305f2ff5465147ee576dd705487c35b15918fcd4efdc0cb7a290" name = "github.com/prometheus/procfs" packages = [ ".", "internal/util", "nfs", - "xfs" + "xfs", ] + pruneopts = "UT" revision = "05ee40e3a273f7245e8777337fc7b46e533a9a92" [[projects]] branch = "master" + digest = "1:c4556a44e350b50a490544d9b06e9fba9c286c21d6c0e47f54f3a9214597298c" name = "github.com/rcrowley/go-metrics" packages = ["."] + pruneopts = "UT" revision = "e2704e165165ec55d062f5919b4b29494e9fa790" [[projects]] + digest = "1:274f67cb6fed9588ea2521ecdac05a6d62a8c51c074c1fccc6a49a40ba80e925" name = "github.com/satori/go.uuid" packages = ["."] + pruneopts = "UT" revision = "f58768cc1a7a7e77a3bd49e98cdd21419399b6a3" version = "v1.2.0" [[projects]] branch = "master" + digest = "1:c1f242537d2898c84fe9bfd8bd94b177cd5ff98a10c397dbbc494f4e2b173be6" name = "github.com/serialx/hashring" packages = ["."] + pruneopts = "UT" revision = "49a4782e9908fe098c907022a1bd7519c79803d6" [[projects]] + digest = "1:d867dfa6751c8d7a435821ad3b736310c2ed68945d05b50fb9d23aee0540c8cc" name = "github.com/sirupsen/logrus" packages = ["."] + pruneopts = "UT" revision = "3e01752db0189b9157070a0e1668a620f9a85da2" version = "v1.0.6" [[projects]] + digest = "1:9424f440bba8f7508b69414634aef3b2b3a877e522d8a4624692412805407bb7" name = "github.com/spf13/pflag" packages = ["."] + pruneopts = "UT" revision = "583c0c0531f06d5278b7d917446061adc344b5cd" version = "v1.0.1" [[projects]] + digest = "1:18752d0b95816a1b777505a97f71c7467a8445b8ffb55631a7bf779f6ba4fa83" name = "github.com/stretchr/testify" packages = ["assert"] + pruneopts = "UT" revision = "f35b8ab0b5a2cef36673838d662e249dd9c94686" version = "v1.2.2" [[projects]] + digest = "1:98e5cda86f67cd1ac95389d98670b66dea8cae480fe6292b83bccccfe60b4106" name = "github.com/ugorji/go" packages = ["codec"] + pruneopts = "UT" revision = "f3cacc17c85ecb7f1b6a9e373ee85d1480919868" [[projects]] branch = "master" + digest = "1:91771429c8de08d4d9b32663ad78bfa8ee1e0a146e921d9937c117665fd0377f" name = "github.com/vmware/govmomi" packages = [ ".", @@ -568,12 +723,14 @@ "vim25/progress", "vim25/soap", "vim25/types", - "vim25/xml" + "vim25/xml", ] + pruneopts = "UT" revision = "bbd99532a768d2fe369079ceda730e30726ae1a6" [[projects]] branch = "master" + digest = "1:014ee502e4880ffba2a0103cefa6af78cf8767ce08bc29376e6b86cdcaa2ec06" name = "golang.org/x/crypto" packages = [ "curve25519", @@ -583,12 +740,14 @@ "internal/subtle", "poly1305", "ssh", - "ssh/terminal" + "ssh/terminal", ] + pruneopts = "UT" revision = "c126467f60eb25f8f27e5a981f32a87e3965053f" [[projects]] branch = "master" + digest = "1:c945d925a3dae287abe075a89c6b09e1015da39d350eb512bb5d01ebcd7783e7" name = "golang.org/x/net" packages = [ "bpf", @@ -602,20 +761,24 @@ "internal/timeseries", "ipv4", "ipv6", - "trace" + "trace", ] + pruneopts = "UT" revision = "f4c29de78a2a91c00474a2e689954305c350adf9" [[projects]] branch = "master" + digest = "1:be95d758fc4d9216e5d41ff5b98b46938fba85ca5bc2ddc45a1b03e2bb10fe7c" name = "golang.org/x/sys" packages = [ "unix", - "windows" + "windows", ] + pruneopts = "UT" revision = "e072cadbbdc8dd3d3ffa82b8b4b9304c261d9311" [[projects]] + digest = "1:a2ab62866c75542dd18d2b069fec854577a20211d7c0ea6ae746072a1dccdd18" name = "golang.org/x/text" packages = [ "collate", @@ -631,30 +794,38 @@ "unicode/bidi", "unicode/cldr", "unicode/norm", - "unicode/rangetable" + "unicode/rangetable", ] + pruneopts = "UT" revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" version = "v0.3.0" [[projects]] branch = "master" + digest = "1:c9e7a4b4d47c0ed205d257648b0e5b0440880cb728506e318f8ac7cd36270bc4" name = "golang.org/x/time" packages = ["rate"] + pruneopts = "UT" revision = "fbb02b2291d28baffd63558aa44b4b56f178d650" [[projects]] + digest = "1:c25289f43ac4a68d88b02245742347c94f1e108c534dda442188015ff80669b3" name = "google.golang.org/appengine" packages = ["cloudsql"] + pruneopts = "UT" revision = "b1f26356af11148e710935ed1ac8a7f5702c7612" version = "v1.1.0" [[projects]] branch = "master" + digest = "1:077c1c599507b3b3e9156d17d36e1e61928ee9b53a5b420f10f28ebd4a0b275c" name = "google.golang.org/genproto" packages = ["googleapis/rpc/status"] + pruneopts = "UT" revision = "daca94659cb50e9f37c1b834680f2e46358f10b0" [[projects]] + digest = "1:3dd7996ce6bf52dec6a2f69fa43e7c4cefea1d4dfa3c8ab7a5f8a9f7434e239d" name = "google.golang.org/grpc" packages = [ ".", @@ -682,36 +853,46 @@ "resolver/passthrough", "stats", "status", - "tap" + "tap", ] + pruneopts = "UT" revision = "32fb0ac620c32ba40a4626ddf94d90d12cce3455" version = "v1.14.0" [[projects]] + digest = "1:2655728699af00d823b0a68bd7169fd241a30c04252f456b34421908701b9c3f" name = "gopkg.in/gin-gonic/gin.v1" packages = ["."] + pruneopts = "UT" revision = "d459835d2b077e44f7c9b453505ee29881d5d12d" version = "v1.2" [[projects]] + digest = "1:cbc72c4c4886a918d6ab4b95e347ffe259846260f99ebdd8a198c2331cf2b2e9" name = "gopkg.in/go-playground/validator.v8" packages = ["."] + pruneopts = "UT" revision = "5f1438d3fca68893a817e4a66806cea46a9e4ebf" version = "v8.18.2" [[projects]] + digest = "1:2d1fbdc6777e5408cabeb02bf336305e724b925ff4546ded0fa8715a7267922a" name = "gopkg.in/inf.v0" packages = ["."] + pruneopts = "UT" revision = "d2d2541c53f18d2a059457998ce2876cc8e67cbf" version = "v0.9.1" [[projects]] + digest = "1:342378ac4dcb378a5448dd723f0784ae519383532f5e70ade24132c4c8693202" name = "gopkg.in/yaml.v2" packages = ["."] + pruneopts = "UT" revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183" version = "v2.2.1" [[projects]] + digest = "1:74142cd2275f77547c35ac51514108d9798a09aa0cf377a5c1084718ef7aa225" name = "k8s.io/api" packages = [ "admissionregistration/v1alpha1", @@ -742,11 +923,13 @@ "settings/v1alpha1", "storage/v1", "storage/v1alpha1", - "storage/v1beta1" + "storage/v1beta1", ] + pruneopts = "UT" revision = "072894a440bdee3a891dea811fe42902311cd2a3" [[projects]] + digest = "1:29d5abc33d8cbec19b4cd032a87d28010dd615578dd695a655fc3d5ec3ab86b6" name = "k8s.io/apimachinery" packages = [ "pkg/api/errors", @@ -783,11 +966,13 @@ "pkg/util/yaml", "pkg/version", "pkg/watch", - "third_party/forked/golang/reflect" + "third_party/forked/golang/reflect", ] + pruneopts = "UT" revision = "103fd098999dc9c0c88536f5c9ad2e5da39373ae" [[projects]] + digest = "1:ac46cc5d5c55dd93d4844327334d2253d563d76f14d4b38a8b522b97f0f52214" name = "k8s.io/client-go" packages = [ "discovery", @@ -841,27 +1026,34 @@ "util/connrotation", "util/flowcontrol", "util/homedir", - "util/integer" + "util/integer", ] + pruneopts = "UT" revision = "7d04d0e2a0a1a4d4a1cd6baa432a2301492e4e65" version = "v8.0.0" [[projects]] branch = "master" + digest = "1:d0257638bb52243f9fa293ef07544081259d54edd8e53f084e522b6c22e92637" name = "yunion.io/x/jsonutils" packages = ["."] - revision = "a86e16793be4a6f4c5ea9203e40c2a0a9361efb8" + pruneopts = "UT" + revision = "38477c9cceb895816fe21507d73da30376d358b7" [[projects]] branch = "master" + digest = "1:636db180d3fc734536a5437d93bb6fe937feb036c1ed062c25872a47c4283163" name = "yunion.io/x/log" packages = [ ".", - "hooks" + "hooks", ] - revision = "a10b94c5038480920262e0f344656328434955a2" + pruneopts = "UT" + revision = "5185c49f6d361f61e561fbccba1bedcc7c62748b" [[projects]] + branch = "master" + digest = "1:8e2a4d71738e5e3bffc415b23f100b0a5033dbf88154783122f5147d4fc213b5" name = "yunion.io/x/pkg" packages = [ "gotypes", @@ -893,25 +1085,122 @@ "util/version", "util/wait", "util/workqueue", - "utils" + "utils", ] - revision = "07cad0cb1df5155b3bfa15079594d01050931554" + pruneopts = "UT" + revision = "7348c84865abac3fcdec618591b7eee7c2e0669b" [[projects]] branch = "master" + digest = "1:e23019db2cda58480528738c86bc7abbff09d10488c2a51dd5d500f8686815f0" name = "yunion.io/x/sqlchemy" packages = ["."] - revision = "f6a47145f9b94ebff1517017e790f068698b5b5a" + pruneopts = "UT" + revision = "19a3c90524d3638f8e3580231396dda46088aebc" [[projects]] branch = "master" + digest = "1:71e1b62648868a9083f9e4693145a0154a600591ca548aaf64cae32a10e956e5" name = "yunion.io/x/structarg" packages = ["."] - revision = "5f84a68c40a2d8eb923f0607f809cbda8d5b944a" + pruneopts = "UT" + revision = "4a5eb8e2cdfbf7f6511561a3c710c116b171f405" [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "63315737a1e02fba65e6ba256ff2571dfb9351af5fe446152e89adaf1c7129fb" + input-imports = [ + "github.com/aliyun/alibaba-cloud-sdk-go/sdk", + "github.com/aliyun/alibaba-cloud-sdk-go/sdk/errors", + "github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests", + "github.com/aliyun/aliyun-oss-go-sdk/oss", + "github.com/aokoli/goutils", + "github.com/bitly/go-simplejson", + "github.com/c-bata/go-prompt", + "github.com/coredns/coredns/core/dnsserver", + "github.com/coredns/coredns/coremain", + "github.com/coredns/coredns/plugin", + "github.com/coredns/coredns/plugin/cache", + "github.com/coredns/coredns/plugin/chaos", + "github.com/coredns/coredns/plugin/debug", + "github.com/coredns/coredns/plugin/errors", + "github.com/coredns/coredns/plugin/etcd/msg", + "github.com/coredns/coredns/plugin/file", + "github.com/coredns/coredns/plugin/forward", + "github.com/coredns/coredns/plugin/health", + "github.com/coredns/coredns/plugin/hosts", + "github.com/coredns/coredns/plugin/log", + "github.com/coredns/coredns/plugin/metrics", + "github.com/coredns/coredns/plugin/nsid", + "github.com/coredns/coredns/plugin/pkg/dnsutil", + "github.com/coredns/coredns/plugin/pkg/fall", + "github.com/coredns/coredns/plugin/pkg/upstream", + "github.com/coredns/coredns/plugin/proxy", + "github.com/coredns/coredns/plugin/reload", + "github.com/coredns/coredns/plugin/trace", + "github.com/coredns/coredns/request", + "github.com/fatih/color", + "github.com/go-sql-driver/mysql", + "github.com/golang-plus/uuid", + "github.com/gosuri/uitable", + "github.com/jinzhu/gorm", + "github.com/jinzhu/gorm/dialects/mysql", + "github.com/json-iterator/go", + "github.com/mholt/caddy", + "github.com/mholt/caddy/startupshutdown", + "github.com/miekg/dns", + "github.com/moul/http2curl", + "github.com/serialx/hashring", + "github.com/stretchr/testify/assert", + "github.com/vmware/govmomi", + "github.com/vmware/govmomi/object", + "github.com/vmware/govmomi/property", + "github.com/vmware/govmomi/session", + "github.com/vmware/govmomi/view", + "github.com/vmware/govmomi/vim25/mo", + "github.com/vmware/govmomi/vim25/types", + "golang.org/x/crypto/ssh", + "gopkg.in/gin-gonic/gin.v1", + "k8s.io/api/core/v1", + "k8s.io/apimachinery/pkg/api/errors", + "k8s.io/apimachinery/pkg/apis/meta/v1", + "k8s.io/apimachinery/pkg/fields", + "k8s.io/apimachinery/pkg/labels", + "k8s.io/client-go/kubernetes", + "k8s.io/client-go/rest", + "k8s.io/client-go/tools/clientcmd", + "yunion.io/x/jsonutils", + "yunion.io/x/log", + "yunion.io/x/pkg/gotypes", + "yunion.io/x/pkg/prettytable", + "yunion.io/x/pkg/trace", + "yunion.io/x/pkg/tristate", + "yunion.io/x/pkg/util/cache", + "yunion.io/x/pkg/util/compare", + "yunion.io/x/pkg/util/errors", + "yunion.io/x/pkg/util/fifoutils", + "yunion.io/x/pkg/util/fileutils", + "yunion.io/x/pkg/util/filterclause", + "yunion.io/x/pkg/util/goroutine_pool", + "yunion.io/x/pkg/util/netutils", + "yunion.io/x/pkg/util/osprofile", + "yunion.io/x/pkg/util/prometheus", + "yunion.io/x/pkg/util/reflectutils", + "yunion.io/x/pkg/util/regutils", + "yunion.io/x/pkg/util/seclib", + "yunion.io/x/pkg/util/secrules", + "yunion.io/x/pkg/util/sets", + "yunion.io/x/pkg/util/stringutils", + "yunion.io/x/pkg/util/sysutils", + "yunion.io/x/pkg/util/timeutils", + "yunion.io/x/pkg/util/trace", + "yunion.io/x/pkg/util/ttlpool", + "yunion.io/x/pkg/util/version", + "yunion.io/x/pkg/util/wait", + "yunion.io/x/pkg/util/workqueue", + "yunion.io/x/pkg/utils", + "yunion.io/x/sqlchemy", + "yunion.io/x/structarg", + ] solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 3343180803..007f3aec01 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -34,7 +34,6 @@ name = "yunion.io/x/jsonutils" [[constraint]] - #revision = "a14641a90b9e5c72868b7a8358d62ea1d83eab53" branch = "master" name = "yunion.io/x/sqlchemy" @@ -43,8 +42,7 @@ name = "yunion.io/x/log" [[constraint]] - revision = "07cad0cb1df5155b3bfa15079594d01050931554" - #branch = "master" + branch = "master" name = "yunion.io/x/pkg" [[constraint]] diff --git a/Makefile b/Makefile index 6537fbd733..ee09d3df04 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ REPO_PREFIX := yunion.io/x/onecloud VENDOR_PATH := $(REPO_PREFIX)/vendor VERSION_PKG := $(VENDOR_PATH)/yunion.io/x/pkg/util/version -ROOT_DIR := $(shell readlink -f `pwd`) +ROOT_DIR := $(CURDIR) BUILD_DIR := $(ROOT_DIR)/_output BIN_DIR := $(BUILD_DIR)/bin BUILD_SCRIPT := $(ROOT_DIR)/build/build.sh @@ -83,10 +83,6 @@ bin_dir: output_dir output_dir: @mkdir -p $(BUILD_DIR) - -dep: - cd $(ROOT_DIR) && dep ensure -v - dep_clean: rm -fr $(GOPATH)/pkg/dep/sources/* @@ -105,5 +101,8 @@ fmt: find . -type f -name "*.go" -not -path "./_output/*" \ -not -path "./vendor/*" | xargs gofmt -s -w +dep: + cd $(ROOT_DIR) && dep ensure -v -update $(shell for p in $$(ls vendor/yunion.io/x/); do echo "yunion.io/x/$$p"; done | xargs) + %: @: diff --git a/build/region-dns/vars b/build/region-dns/vars new file mode 100644 index 0000000000..778d49a5da --- /dev/null +++ b/build/region-dns/vars @@ -0,0 +1,2 @@ +DESCRIPTION="Yunion Cloud Region DNS Service" +# SERVICE="yes" diff --git a/cmd/climc/climc.go b/cmd/climc/climc.go index e5983068a6..a8cc818dd9 100644 --- a/cmd/climc/climc.go +++ b/cmd/climc/climc.go @@ -5,28 +5,32 @@ import ( "fmt" "io/ioutil" "os" + "path/filepath" "strings" "time" prompt "github.com/c-bata/go-prompt" + "yunion.io/x/log" "yunion.io/x/pkg/util/version" "yunion.io/x/structarg" "yunion.io/x/onecloud/cmd/climc/promputils" "yunion.io/x/onecloud/cmd/climc/shell" - _ "yunion.io/x/onecloud/cmd/climc/shell/k8s" "yunion.io/x/onecloud/pkg/mcclient" + + _ "yunion.io/x/onecloud/cmd/climc/shell/k8s" ) type BaseOptions struct { - Help bool `help:"Show help" short-token:"h"` - Debug bool `help:"Show debug information"` - Version bool `help:"Show version"` - Timeout int `default:"600" help:"Number of seconds to wait for a response"` - Secure bool `default:"False" help:"do server cert verification if URL is https"` - OsUsername string `default:"$OS_USERNAME" help:"Username, defaults to env[OS_USERNAME]"` - OsPassword string `default:"$OS_PASSWORD" help:"Password, defaults to env[OS_PASSWORD]"` + Help bool `help:"Show help" short-token:"h"` + Debug bool `help:"Show debug information"` + Version bool `help:"Show version"` + Timeout int `default:"600" help:"Number of seconds to wait for a response"` + Insecure bool `default:"false" help:"Allow skip server cert verification if URL is https" short-token:"k"` + NoCachedToken bool `default:"false" help:"Force not use cached token"` + OsUsername string `default:"$OS_USERNAME" help:"Username, defaults to env[OS_USERNAME]"` + OsPassword string `default:"$OS_PASSWORD" help:"Password, defaults to env[OS_PASSWORD]"` // OsProjectId string `default:"$OS_PROJECT_ID" help:"Proejct ID, defaults to env[OS_PROJECT_ID]"` OsProjectName string `default:"$OS_PROJECT_NAME" help:"Project name, defaults to env[OS_PROJECT_NAME]"` OsDomainName string `default:"$OS_DOMAIN_NAME" help:"Domain name, defaults to env[OS_DOMAIN_NAME]"` @@ -119,11 +123,12 @@ func newClientSession(options *BaseOptions) (*mcclient.ClientSession, error) { client := mcclient.NewClient(options.OsAuthURL, options.Timeout, options.Debug, - options.Secure) + options.Insecure) var cacheToken mcclient.TokenCredential - cacheFile, err := os.Open("/tmp/OS_AUTH_CACHE_TOKEN") - if err == nil && cacheFile != nil { + tokenCachePath := filepath.Join(os.TempDir(), "OS_AUTH_CACHE_TOKEN") + cacheFile, err := os.Open(tokenCachePath) + if err == nil && cacheFile != nil && !options.NoCachedToken { fileInfo, _ := cacheFile.Stat() dur, err := time.ParseDuration("-24h") if fileInfo != nil && err == nil && fileInfo.ModTime().After(time.Now().Add(dur)) { @@ -153,12 +158,12 @@ func newClientSession(options *BaseOptions) (*mcclient.ClientSession, error) { if err != nil { fmt.Printf("Marshal token error:%s", err) } else { - fo, _ := os.Create("/tmp/OS_AUTH_CACHE_TOKEN") + fo, _ := os.Create(tokenCachePath) fo.Write(bytesCacheToken) fo.Close() } } else { - fmt.Println("******** Use Token Cache At /tmp/OS_AUTH_CACHE_TOKEN ********") + fmt.Printf("******** Use Token Cache At %s ********\n", tokenCachePath) } session := client.NewSession(options.OsRegionName, @@ -169,6 +174,40 @@ func newClientSession(options *BaseOptions) (*mcclient.ClientSession, error) { return session, nil } +func enterInteractiveMode( + parser *structarg.ArgumentParser, + sessionFactory func() *mcclient.ClientSession, +) { + promputils.InitEnv(parser, sessionFactory()) + defer fmt.Println("Bye!") + p := prompt.New( + promputils.Executor, + promputils.Completer, + prompt.OptionPrefix("climc> "), + prompt.OptionTitle("Climc, a Command Line Interface to Manage Clouds"), + prompt.OptionMaxSuggestion(16), + ) + p.Run() +} + +func executeSubcommand( + subcmd *structarg.SubcommandArgument, + subparser *structarg.ArgumentParser, + options *BaseOptions, + sessionFactory func() *mcclient.ClientSession, +) { + var e error + suboptions := subparser.Options() + if options.SUBCOMMAND == "help" { + e = subcmd.Invoke(suboptions) + } else { + e = subcmd.Invoke(sessionFactory(), suboptions) + } + if e != nil { + showErrorAndExit(e) + } +} + func main() { parser, e := getSubcommandsParser() if e != nil { @@ -179,47 +218,39 @@ func main() { if options.Help { fmt.Print(parser.HelpString()) - } else if options.Version { - fmt.Printf("Yunion API client version:\n %s\n", version.GetJsonString()) - } else if len(options.SUBCOMMAND) == 0 { - session, e := newClientSession(options) - if e != nil { - showErrorAndExit(e) - } - promputils.InitEnv(parser, session) - defer fmt.Println("Bye!") - p := prompt.New( - promputils.Executor, - promputils.Completer, - prompt.OptionPrefix("climc> "), - prompt.OptionTitle("Climc, a Command Line Interface to Manage Clouds"), - prompt.OptionMaxSuggestion(16), - ) - p.Run() - } else { - subcmd := parser.GetSubcommand() - subparser := subcmd.GetSubParser() - if e != nil { - if subparser != nil { - fmt.Print(subparser.Usage()) - } else { - fmt.Print(parser.Usage()) - } - showErrorAndExit(e) - } else { - session, e := newClientSession(options) - if e != nil { - showErrorAndExit(e) - } - suboptions := subparser.Options() - if options.SUBCOMMAND == "help" { - e = subcmd.Invoke(suboptions) - } else { - e = subcmd.Invoke(session, suboptions) - } - if e != nil { - showErrorAndExit(e) - } - } + return } + + if options.Version { + fmt.Printf("Yunion API client version:\n %s\n", version.GetJsonString()) + return + } + + ensureSessionFactory := func() *mcclient.ClientSession { + session, err := newClientSession(options) + if err != nil { + showErrorAndExit(err) + } + return session + } + + // enter interactive mode when not enough argument and SUBCOMMAND is empty + if _, ok := e.(*structarg.NotEnoughArgumentsError); ok && options.SUBCOMMAND == "" { + enterInteractiveMode(parser, ensureSessionFactory) + return + } + + subcmd := parser.GetSubcommand() + subparser := subcmd.GetSubParser() + if e != nil { + if subparser != nil { + fmt.Print(subparser.Usage()) + } else { + fmt.Print(parser.Usage()) + } + showErrorAndExit(e) + } + + // execute subcommand in non-interactive mode + executeSubcommand(subcmd, subparser, options, ensureSessionFactory) } diff --git a/cmd/climc/promputils/executor.go b/cmd/climc/promputils/executor.go index 9047dd12ce..2b48225b17 100644 --- a/cmd/climc/promputils/executor.go +++ b/cmd/climc/promputils/executor.go @@ -9,6 +9,7 @@ import ( "regexp" "strings" + "yunion.io/x/pkg/utils" "yunion.io/x/structarg" "yunion.io/x/onecloud/pkg/mcclient" @@ -51,8 +52,8 @@ func Executor(s string) { return } s = escaper(s) - args := strings.Split(s, " ") - e := parser.ParseArgs(strings.Split(s, " "), false) + args := utils.ArgsStringToArray(s) + e := parser.ParseArgs(utils.ArgsStringToArray(s), false) subcmd := parser.GetSubcommand() subparser := subcmd.GetSubParser() if args[0] == "--debug" { diff --git a/cmd/climc/shell/cloudproviders.go b/cmd/climc/shell/cloudproviders.go index edfb638d90..36c986ddf2 100644 --- a/cmd/climc/shell/cloudproviders.go +++ b/cmd/climc/shell/cloudproviders.go @@ -140,12 +140,12 @@ func init() { }) type CloudproviderSyncOptions struct { - ID string `help:"ID or Name of cloud provider"` - Force bool `help:"Force sync no matter what"` - FullSync bool `help:"Synchronize everything"` - Region []string `help:"region to sync"` - Zone []string `help:"region to sync"` - Host []string `help:"region to sync"` + ID string `help:"ID or Name of cloud provider"` + Force bool `help:"Force sync no matter what"` + FullSync bool `help:"Synchronize everything"` + Region []string `help:"region to sync"` + Zone []string `help:"region to sync"` + Host []string `help:"region to sync"` } R(&CloudproviderSyncOptions{}, "cloud-provider-sync", "Sync of a cloud provider account", func(s *mcclient.ClientSession, args *CloudproviderSyncOptions) error { params := jsonutils.NewDict() diff --git a/cmd/climc/shell/dnsrecords.go b/cmd/climc/shell/dnsrecords.go index 45b7c923ff..05185894ef 100644 --- a/cmd/climc/shell/dnsrecords.go +++ b/cmd/climc/shell/dnsrecords.go @@ -10,12 +10,12 @@ import ( ) type DNSRecordOptions struct { - A []string `help:"DNS A record" metavar:"A_RECORD" optional:"true"` - AAAA []string `help:"DNS AAAA record" metavar:"AAAA_RECORD" optional:"true"` - CNAME string `help:"DNS CNAME record" metavar:"CNAME_RECORD" optional:"true"` - SRVHost string `help:"DNS SRV record, server of service" metavar:"SRV_RECORD_HOST" optional:"true"` - SRVPort int64 `help:"DNS SRV record, port of service" metavar:"SRV_RECORD_PORT" optional:"true"` - PTR string `help:"DNS PTR record" metavar:"PTR_RECORD" optional:"true"` + A []string `help:"DNS A record" metavar:"A_RECORD" positional:"false"` + AAAA []string `help:"DNS AAAA record" metavar:"AAAA_RECORD" positional:"false"` + CNAME string `help:"DNS CNAME record" metavar:"CNAME_RECORD" positional:"false"` + SRVHost string `help:"DNS SRV record, server of service" metavar:"SRV_RECORD_HOST" positional:"false"` + SRVPort int64 `help:"DNS SRV record, port of service" metavar:"SRV_RECORD_PORT" positional:"false"` + PTR string `help:"DNS PTR record" metavar:"PTR_RECORD" positional:"false"` } func parseDNSRecords(args *DNSRecordOptions, params *jsonutils.JSONDict) { @@ -52,7 +52,7 @@ func init() { type DNSCreateOptions struct { NAME string `help:"DNS name to create"` - TTL int64 `help:"TTL in seconds" optional:"true"` + TTL int64 `help:"TTL in seconds" positional:"false"` Desc string `help:"Description" metavar:"DESCRIPTION"` DNSRecordOptions } @@ -92,7 +92,7 @@ func init() { type DNSUpdateOptions struct { ID string `help:"ID of DNS record to update"` Name string `help:"Domain name"` - TTL int64 `help:"TTL in seconds" optional:"true"` + TTL int64 `help:"TTL in seconds" positional:"false"` Desc string `help:"Description"` } R(&DNSUpdateOptions{}, "dns-update", "Update details of a dns records", func(s *mcclient.ClientSession, args *DNSUpdateOptions) error { diff --git a/cmd/climc/shell/k8s/charts.go b/cmd/climc/shell/k8s/charts.go new file mode 100644 index 0000000000..8cb609642a --- /dev/null +++ b/cmd/climc/shell/k8s/charts.go @@ -0,0 +1,63 @@ +package k8s + +import ( + "fmt" + + json "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules/k8s" +) + +func initChart() { + cmdN := func(suffix string) string { + return resourceCmdN("chart", suffix) + } + + type listOpt struct { + baseListOptions + Name string `help:"Chart name"` + Repo string `help:"Repository name"` + RepoUrl string `help:"Repository url"` + AllVersion bool `json:"Get Chart all history versions"` + Keyword string `json:"Chart keyword"` + } + R(&listOpt{}, cmdN("list"), "List k8s helm global charts", func(s *mcclient.ClientSession, args *listOpt) error { + params := fetchPagingParams(args.baseListOptions) + if len(args.Name) != 0 { + params.Add(json.NewString(args.Name), "name") + } + if len(args.Repo) != 0 { + params.Add(json.NewString(args.Repo), "repo") + } + if len(args.RepoUrl) != 0 { + params.Add(json.NewString(args.RepoUrl), "repo_url") + } + if args.AllVersion { + params.Add(json.JSONTrue, "all_version") + } + if len(args.Keyword) != 0 { + params.Add(json.NewString(args.Keyword), "keyword") + } + charts, err := k8s.Charts.List(s, params) + if err != nil { + return err + } + PrintHelmListResult(charts, ChartPrinter{}) + return nil + }) +} + +type ChartPrinter struct{} + +func (p ChartPrinter) Columns() []interface{} { + return []interface{}{"REPO/NAME", "VERSION", "DESCRIPTION"} +} + +func (p ChartPrinter) RowKeys(obj *json.JSONDict) []interface{} { + repo, _ := obj.GetString("repo") + name, _ := obj.GetString("chart", "metadata", "name") + version, _ := obj.GetString("chart", "metadata", "version") + desc, _ := obj.GetString("chart", "metadata", "description") + return []interface{}{fmt.Sprintf("%s/%s", repo, name), version, desc} +} diff --git a/cmd/climc/shell/k8s/configmap.go b/cmd/climc/shell/k8s/configmap.go new file mode 100644 index 0000000000..30d8fd6d9e --- /dev/null +++ b/cmd/climc/shell/k8s/configmap.go @@ -0,0 +1,27 @@ +package k8s + +import ( + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules/k8s" +) + +func initConfigMap() { + cmdN := func(suffix string) string { + return resourceCmdN("configmap", suffix) + } + + type listOpt struct { + namespaceListOptions + baseListOptions + } + R(&listOpt{}, cmdN("list"), "List k8s configmap", func(s *mcclient.ClientSession, args *listOpt) error { + params := fetchNamespaceParams(args.namespaceListOptions) + params.Update(fetchPagingParams(args.baseListOptions)) + ret, err := k8s.ConfigMaps.ListInContexts(s, params, args.ClusterContext()) + if err != nil { + return err + } + printList(ret, k8s.ConfigMaps.GetColumns(s)) + return nil + }) +} diff --git a/cmd/climc/shell/k8s/deployment.go b/cmd/climc/shell/k8s/deployment.go new file mode 100644 index 0000000000..dfdc392bba --- /dev/null +++ b/cmd/climc/shell/k8s/deployment.go @@ -0,0 +1,145 @@ +package k8s + +import ( + "fmt" + "strconv" + "strings" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/util/sets" + + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules/k8s" +) + +func initDeployment() { + cmdN := func(suffix string) string { + return resourceCmdN("deployment", suffix) + } + + type listOpt struct { + namespaceListOptions + baseListOptions + } + R(&listOpt{}, cmdN("list"), "List k8s deployment", func(s *mcclient.ClientSession, args *listOpt) error { + params := fetchNamespaceParams(args.namespaceListOptions) + params.Update(fetchPagingParams(args.baseListOptions)) + ret, err := k8s.Deployments.ListInContexts(s, params, args.ClusterContext()) + if err != nil { + return err + } + printList(ret, k8s.Deployments.GetColumns(s)) + return nil + }) + + type createOpt struct { + namespaceOptions + NAME string `help:"Name of deployment"` + Image string `help:"The image for the container to run"` + Replicas int64 `help:"Number of replicas for pods in this deployment"` + RunAsPrivileged bool `help:"Whether to run the container as privileged user"` + Labels string `help:"Comma separated labels to apply to the pod(s)"` + Env []string `help:"Environment variables to set in container"` + Port []string `help:"Port for the service that is created, format is :: e.g. tcp:80:3000"` + } + R(&createOpt{}, cmdN("create"), "Create deployment resource", func(s *mcclient.ClientSession, args *createOpt) error { + params := jsonutils.NewDict() + if len(args.Image) == 0 { + return fmt.Errorf("Image must provided") + } + params.Add(jsonutils.NewString(args.NAME), "name") + params.Add(jsonutils.NewString(args.Image), "containerImage") + if args.Namespace != "" { + params.Add(jsonutils.NewString(args.Namespace), "namespace") + } + if args.Replicas > 1 { + params.Add(jsonutils.NewInt(args.Replicas), "replicas") + } + if args.RunAsPrivileged { + params.Add(jsonutils.JSONTrue, "runAsPrivileged") + } + if len(args.Port) != 0 { + portMappings, err := parsePortMappings(args.Port) + if err != nil { + return err + } + params.Add(portMappings, "portMappings") + } + ret, err := k8s.Deployments.CreateInContexts(s, params, args.ClusterContext()) + if err != nil { + return err + } + printObject(ret) + return nil + }) + + type getOpt struct { + resourceGetOptions + } + R(&getOpt{}, cmdN("show"), "Get deployment details", func(s *mcclient.ClientSession, args *getOpt) error { + id := args.NAME + params := jsonutils.NewDict() + if args.Namespace != "" { + params.Add(jsonutils.NewString(args.Namespace), "namespace") + } + ret, err := k8s.Deployments.GetInContexts(s, id, params, args.ClusterContext()) + if err != nil { + return err + } + printObjectYAML(ret) + return nil + }) +} + +type portMapping struct { + Port int32 `json:"port"` + TargetPort int32 `json:"targetPort"` + Protocol string `json:"protocol"` +} + +func parsePortMapping(port string) (*portMapping, error) { + if len(port) == 0 { + return nil, fmt.Errorf("empty port mapping desc string") + } + parts := strings.Split(port, ":") + mapping := &portMapping{} + for _, part := range parts { + if sets.NewString("tcp", "udp").Has(strings.ToLower(part)) { + mapping.Protocol = strings.ToUpper(part) + } + if port, err := strconv.Atoi(part); err != nil { + continue + } else { + if mapping.Port == 0 { + mapping.Port = int32(port) + } else { + mapping.TargetPort = int32(port) + } + } + } + if mapping.Protocol == "" { + mapping.Protocol = "TCP" + } + if mapping.Port <= 0 { + return nil, fmt.Errorf("Service port not provided") + } + if mapping.TargetPort < 0 { + return nil, fmt.Errorf("Container invalid targetPort %d", mapping.TargetPort) + } + if mapping.TargetPort == 0 { + mapping.TargetPort = mapping.Port + } + return mapping, nil +} + +func parsePortMappings(ports []string) (*jsonutils.JSONArray, error) { + ret := jsonutils.NewArray() + for _, port := range ports { + mapping, err := parsePortMapping(port) + if err != nil { + return nil, fmt.Errorf("Port %q error: %v", port, err) + } + ret.Add(jsonutils.Marshal(mapping)) + } + return ret, nil +} diff --git a/cmd/climc/shell/k8s/helm_list.go b/cmd/climc/shell/k8s/helm_list.go new file mode 100644 index 0000000000..df1055a29e --- /dev/null +++ b/cmd/climc/shell/k8s/helm_list.go @@ -0,0 +1,51 @@ +package k8s + +import ( + "fmt" + + "github.com/gosuri/uitable" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +type ListPrinter interface { + Columns() []interface{} + RowKeys(obj *jsonutils.JSONDict) []interface{} +} + +func ListerTable(res *modules.ListResult, printer ListPrinter) *uitable.Table { + min := func(x, y int) int { + if x < y { + return x + } + return y + } + table := uitable.New() + table.MaxColWidth = 80 + table.AddRow(printer.Columns()...) + var idx int + for ; idx < min(res.Limit, res.Total); idx++ { + table.AddRow(printer.RowKeys(res.Data[idx].(*jsonutils.JSONDict))...) + } + return table +} + +func PrintHelmListResult(res *modules.ListResult, printer ListPrinter) { + fmt.Println(ListerTable(res, printer)) + + table := uitable.New() + total := res.Total + offset := res.Offset + limit := res.Limit + page := (offset / limit) + 1 + pages := total / limit + if pages*limit < total { + pages += 1 + } + table.AddRow("") + table.AddRow("Total", "Pages", "Limit", "Offset", "Page") + table.AddRow(total, pages, limit, offset, page) + fmt.Println(table) +} diff --git a/cmd/climc/shell/k8s/k8s.go b/cmd/climc/shell/k8s/k8s.go index 2e63757294..d7b46f482a 100644 --- a/cmd/climc/shell/k8s/k8s.go +++ b/cmd/climc/shell/k8s/k8s.go @@ -12,24 +12,84 @@ import ( ) func init() { + // cluster resources initCluster() initNode() + + // helm resources + initTiller() + initRepo() + initChart() + initRelease() + + // kubernetes original resources + initRaw() + initConfigMap() + initDeployment() initPod() + initService() } type BaseListOptions shell.BaseListOptions type clusterBaseOptions struct { - Cluster string `default:"$K8S_CLUSTER" help:"Kubernetes cluster name"` + Cluster string `default:"$K8S_CLUSTER|default" help:"Kubernetes cluster name"` } func (o clusterBaseOptions) ClusterContext() []modules.ManagerContext { return []modules.ManagerContext{clusterContext(o.Cluster)} } -type k8sBaseListOptions struct { +type baseListOptions struct { + Limit int `default:"20" help:"Page limit"` + Offset int `default:"0" help:"page offset"` +} + +func fetchPagingParams(opt baseListOptions) *jsonutils.JSONDict { + params := jsonutils.NewDict() + if opt.Limit > 0 { + params.Add(jsonutils.NewInt(int64(opt.Limit)), "limit") + } + if opt.Offset > 0 { + params.Add(jsonutils.NewInt(int64(opt.Offset)), "offset") + } + return params +} + +type namespaceListOptions struct { + namespaceOptions + AllNamespace bool `help:"Show resource in all namespace"` +} + +type namespaceOptions struct { clusterBaseOptions - Limit int `default:"20" help:"Page limit"` + Namespace string `help:"Namespace of this resource"` +} + +type resourceGetOptions struct { + clusterBaseOptions + Namespace string `help:"Namespace of this resource"` + NAME string `help:"Name ident of the resource"` +} + +func (o resourceGetOptions) ToJSON() *jsonutils.JSONDict { + params := jsonutils.NewDict() + if o.Namespace != "" { + params.Add(jsonutils.NewString(o.Namespace), "namespace") + } + return params +} + +func fetchNamespaceParams(opt namespaceListOptions) *jsonutils.JSONDict { + params := jsonutils.NewDict() + if opt.AllNamespace { + params.Add(jsonutils.JSONTrue, "all_namespace") + return params + } + if opt.Namespace != "" { + params.Add(jsonutils.NewString(opt.Namespace), "namespace") + } + return params } var ( @@ -53,3 +113,7 @@ func clusterContext(clusterId string) modules.ManagerContext { InstanceId: clusterId, } } + +func printObjectYAML(obj jsonutils.JSONObject) { + fmt.Println(obj.YAMLString()) +} diff --git a/cmd/climc/shell/k8s/pods.go b/cmd/climc/shell/k8s/pods.go index 8d2fdd0d3b..ee8803c414 100644 --- a/cmd/climc/shell/k8s/pods.go +++ b/cmd/climc/shell/k8s/pods.go @@ -13,10 +13,26 @@ func initPod() { } type listOpt struct { - k8sBaseListOptions + namespaceListOptions + baseListOptions } R(&listOpt{}, cmdN("list"), "List k8s pod", func(s *mcclient.ClientSession, args *listOpt) error { - ret, err := k8s.Pods.ListInContexts(s, nil, args.ClusterContext()) + params := fetchNamespaceParams(args.namespaceListOptions) + params.Update(fetchPagingParams(args.baseListOptions)) + ret, err := k8s.Pods.ListInContexts(s, params, args.ClusterContext()) + if err != nil { + return err + } + printList(ret, k8s.Pods.GetColumns(s)) + return nil + }) + + type deleteOpt struct { + resourceGetOptions + } + R(&deleteOpt{}, cmdN("delete"), "Delete pod", func(s *mcclient.ClientSession, args *deleteOpt) error { + id := args.NAME + ret, err := k8s.Pods.DeleteInContexts(s, id, args.ToJSON(), args.ClusterContext()) if err != nil { return err } diff --git a/cmd/climc/shell/k8s/raw.go b/cmd/climc/shell/k8s/raw.go new file mode 100644 index 0000000000..6aeb500f91 --- /dev/null +++ b/cmd/climc/shell/k8s/raw.go @@ -0,0 +1,35 @@ +package k8s + +import ( + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules/k8s" +) + +type rawGetOpt struct { + namespaceOptions + KIND string `help:"resource kind"` + NAME string `help:"instance name"` +} + +type rawDeleteOpt struct { + rawGetOpt +} + +func initRaw() { + R(&rawGetOpt{}, "k8s-get", "Get k8s resource instance raw info", func(s *mcclient.ClientSession, args *rawGetOpt) error { + obj, err := k8s.RawResource.Get(s, args.KIND, args.Namespace, args.NAME, nil, args.ClusterContext()) + if err != nil { + return err + } + printObjectYAML(obj) + return nil + }) + + R(&rawDeleteOpt{}, "k8s-delete", "Delete k8s resource instance", func(s *mcclient.ClientSession, args *rawDeleteOpt) error { + err := k8s.RawResource.Delete(s, args.KIND, args.Namespace, args.NAME, nil, args.ClusterContext()) + if err != nil { + return err + } + return nil + }) +} diff --git a/cmd/climc/shell/k8s/releases.go b/cmd/climc/shell/k8s/releases.go new file mode 100644 index 0000000000..179fb199b4 --- /dev/null +++ b/cmd/climc/shell/k8s/releases.go @@ -0,0 +1,177 @@ +package k8s + +import ( + "fmt" + "io/ioutil" + + json "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules/k8s" +) + +func initRelease() { + cmdN := func(suffix string) string { + return resourceCmdN("release", suffix) + } + type listOpt struct { + namespaceListOptions + baseListOptions + Filter string `help:"Filter, split by space"` + Admin bool `help:"Admin to show all namespace releases"` + Deployed bool `help:"Show deployed status releases"` + Deleted bool `help:"Show deleted status releases"` + Deleting bool `help:"Show deleting status releases"` + Failed bool `help:"Show failed status releases"` + Superseded bool `help:"Show superseded status releases"` + Pending bool `help:"Show pending status releases"` + } + R(&listOpt{}, cmdN("list"), "List k8s cluster helm releases", func(s *mcclient.ClientSession, args *listOpt) error { + params := fetchNamespaceParams(args.namespaceListOptions) + params.Update(fetchPagingParams(args.baseListOptions)) + if args.Filter != "" { + params.Add(json.NewString(args.Filter), "filter") + } + if args.Namespace != "" { + params.Add(json.NewString(args.Namespace), "namespace") + } + params.Add(json.JSONTrue, "all") + if args.Admin { + params.Add(json.JSONTrue, "admin") + } + if args.Deployed { + params.Add(json.JSONTrue, "deployed") + } + if args.Deleted { + params.Add(json.JSONTrue, "deleted") + } + if args.Deleting { + params.Add(json.JSONTrue, "deleting") + } + if args.Failed { + params.Add(json.JSONTrue, "failed") + } + if args.Superseded { + params.Add(json.JSONTrue, "superseded") + } + if args.Pending { + params.Add(json.JSONTrue, "pending") + } + ret, err := k8s.Releases.ListInContexts(s, params, args.ClusterContext()) + if err != nil { + return err + } + printList(ret, k8s.Releases.GetColumns(s)) + return nil + }) + + type showOpt struct { + clusterBaseOptions + NAME string `help:"Release instance name"` + } + R(&showOpt{}, cmdN("show"), "Get helm release details", func(s *mcclient.ClientSession, args *showOpt) error { + ret, err := k8s.Releases.GetInContexts(s, args.NAME, nil, args.ClusterContext()) + if err != nil { + return err + } + resources, err := ret.GetString("info", "status", "resources") + if err != nil { + return err + } + fmt.Println(resources) + return nil + }) + + type releaseCUOpts struct { + Values string `help:"Specify values in a YAML file (can specify multiple)" short-token:"f"` + Version string `help:"Specify the exact chart version to install. If not specified, latest version installed"` + //Set []string `help:"set STRING values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)"` + DryRun bool `help:"Simulate an install"` + Details bool `help:"Show release deploy details, include kubernetes created resources"` + Timeout int64 `help:"Time in seconds to wait for any individual kubernetes operation (like Jobs for hooks)" default:"600"` + } + releaseCUDict := func(args releaseCUOpts) (*json.JSONDict, error) { + params := json.NewDict() + if args.Version != "" { + params.Add(json.NewString(args.Version), "version") + } + if args.DryRun { + params.Add(json.JSONTrue, "dry_run") + } + params.Add(json.NewInt(args.Timeout), "timeout") + if args.Values != "" { + //vals, err := helm.MergeValuesF(args.Values, args.Set, []string{}) + vals, err := ioutil.ReadFile(args.Values) + if err != nil { + return nil, err + } + params.Add(json.NewString(string(vals)), "values") + } + return params, nil + } + + type releaseCreateOpts struct { + namespaceOptions + releaseCUOpts + Name string `help:"Release name, If unspecified, it will autogenerate one for you"` + CHARTNAME string `help:"Helm chart name, e.g stable/etcd"` + } + R(&releaseCreateOpts{}, cmdN("create"), "Create release with specified helm chart", func(s *mcclient.ClientSession, args *releaseCreateOpts) error { + params, err := releaseCUDict(args.releaseCUOpts) + if err != nil { + return err + } + params.Add(json.NewString(args.CHARTNAME), "chart_name") + if args.Namespace != "" { + params.Add(json.NewString(args.Namespace), "namespace") + } + if args.Name != "" { + params.Add(json.NewString(args.Name), "release_name") + } + ret, err := k8s.Releases.CreateInContexts(s, params, args.ClusterContext()) + if err != nil { + return err + } + printObject(ret) + return nil + }) + + type releaseUpgradeOpts struct { + clusterBaseOptions + releaseCUOpts + NAME string `help:"Release instance name"` + CHARTNAME string `help:"Helm chart name, e.g stable/etcd"` + ReuseValues bool `help:"When upgrading, reuse the last release's values, and merge in any new values. If '--reset-values' is specified, this is ignored"` + ResetValues bool `help:"When upgrading, reset the values to the ones built into the chart"` + } + R(&releaseUpgradeOpts{}, cmdN("upgrade"), "Upgrade release", func(s *mcclient.ClientSession, args *releaseUpgradeOpts) error { + params, err := releaseCUDict(args.releaseCUOpts) + if err != nil { + return err + } + params.Add(json.NewString(args.CHARTNAME), "chart_name") + params.Add(json.NewString(args.NAME), "release_name") + if args.ReuseValues { + params.Add(json.JSONTrue, "reuse_values") + } + if args.ResetValues { + params.Add(json.JSONTrue, "reset_values") + } + + res, err := k8s.Releases.PutInContexts(s, args.NAME, params, args.ClusterContext()) + if err != nil { + return err + } + printObject(res) + return nil + }) + + type deleteOpt struct { + clusterBaseOptions + NAME string `help:"Release instance name"` + } + R(&deleteOpt{}, cmdN("delete"), "Delete release", func(s *mcclient.ClientSession, args *deleteOpt) error { + _, err := k8s.Releases.DeleteInContexts(s, args.NAME, nil, args.ClusterContext()) + return err + }) +} diff --git a/cmd/climc/shell/k8s/repos.go b/cmd/climc/shell/k8s/repos.go new file mode 100644 index 0000000000..ee34d8a2f9 --- /dev/null +++ b/cmd/climc/shell/k8s/repos.go @@ -0,0 +1,106 @@ +package k8s + +import ( + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules/k8s" +) + +func initRepo() { + cmdN := func(suffix string) string { + return resourceCmdN("repo", suffix) + } + type listOpt struct { + BaseListOptions + } + R(&listOpt{}, cmdN("list"), "List k8s global helm repos", func(s *mcclient.ClientSession, args *listOpt) error { + params := FetchPagingParams(args.BaseListOptions) + result, err := k8s.Repos.List(s, params) + if err != nil { + return err + } + printList(result, k8s.Repos.GetColumns(s)) + return nil + }) + + type getOpt struct { + NAME string `help:"ID or name of the repo"` + } + R(&getOpt{}, cmdN("show"), "Show details fo a repo", func(s *mcclient.ClientSession, args *getOpt) error { + repo, err := k8s.Repos.Get(s, args.NAME, nil) + if err != nil { + return err + } + printObject(repo) + return nil + }) + + type createOpt struct { + getOpt + URL string `help:"Repository url"` + Public bool `help:"Make repository public"` + } + R(&createOpt{}, cmdN("create"), "Add repository", func(s *mcclient.ClientSession, args *createOpt) error { + params := jsonutils.NewDict() + params.Add(jsonutils.NewString(args.NAME), "name") + params.Add(jsonutils.NewString(args.URL), "url") + if args.Public { + params.Add(jsonutils.JSONTrue, "is_public") + } + repo, err := k8s.Repos.Create(s, params) + if err != nil { + return err + } + printObject(repo) + return nil + }) + + type updateOpt struct { + getOpt + Name string `help:"Repository name to change"` + Url string `help:"Repository url to change"` + } + R(&updateOpt{}, cmdN("update"), "Update helm repository", func(s *mcclient.ClientSession, args *updateOpt) error { + params := jsonutils.NewDict() + if args.Name != "" { + params.Add(jsonutils.NewString(args.Name), "name") + } + if args.Url != "" { + params.Add(jsonutils.NewString(args.Url), "url") + } + repo, err := k8s.Repos.Update(s, args.NAME, params) + if err != nil { + return err + } + printObject(repo) + return nil + }) + + R(&getOpt{}, cmdN("private"), "Make repository private", func(s *mcclient.ClientSession, args *getOpt) error { + repo, err := k8s.Repos.PerformAction(s, args.NAME, "private", nil) + if err != nil { + return err + } + printObject(repo) + return nil + }) + + R(&getOpt{}, cmdN("public"), "Make repository public", func(s *mcclient.ClientSession, args *getOpt) error { + repo, err := k8s.Repos.PerformAction(s, args.NAME, "public", nil) + if err != nil { + return err + } + printObject(repo) + return nil + }) + + R(&getOpt{}, cmdN("delete"), "Delete a repository", func(s *mcclient.ClientSession, args *getOpt) error { + repo, err := k8s.Repos.Delete(s, args.NAME, nil) + if err != nil { + return err + } + printObject(repo) + return nil + }) +} diff --git a/cmd/climc/shell/k8s/service.go b/cmd/climc/shell/k8s/service.go new file mode 100644 index 0000000000..3260b28b88 --- /dev/null +++ b/cmd/climc/shell/k8s/service.go @@ -0,0 +1,27 @@ +package k8s + +import ( + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules/k8s" +) + +func initService() { + cmdN := func(suffix string) string { + return resourceCmdN("service", suffix) + } + + type listOpt struct { + namespaceListOptions + baseListOptions + } + R(&listOpt{}, cmdN("list"), "List k8s service", func(s *mcclient.ClientSession, args *listOpt) error { + params := fetchNamespaceParams(args.namespaceListOptions) + params.Update(fetchPagingParams(args.baseListOptions)) + ret, err := k8s.Services.ListInContexts(s, params, args.ClusterContext()) + if err != nil { + return err + } + printList(ret, k8s.Services.GetColumns(s)) + return nil + }) +} diff --git a/cmd/climc/shell/k8s/tiller.go b/cmd/climc/shell/k8s/tiller.go new file mode 100644 index 0000000000..247e5fba13 --- /dev/null +++ b/cmd/climc/shell/k8s/tiller.go @@ -0,0 +1,56 @@ +package k8s + +import ( + json "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules/k8s" +) + +func initTiller() { + cmdN := func(suffix string) string { + return resourceCmdN("tiller", suffix) + } + type createOpt struct { + clusterBaseOptions + KubeContext string `json:"kube_context"` + Namespace string `json:"namespace" default:"kube-system"` + // Upgrade if Tiller is already installed + Upgrade bool `json:"upgrade"` + // Name of service account + ServiceAccount string `json:"service_account" default:"tiller"` + // Use the canary Tiller image + Canary bool `json:"canary_image"` + + // Override Tiller image + Image string `json:"tiller_image" default:"yunion/tiller:v2.9.0"` + // Limit the maximum number of revisions saved per release. Use 0 for no limit. + MaxHistory int `json:"history_max"` + } + R(&createOpt{}, cmdN("create"), "Install helm tiller server to Kubernetes cluster", func(s *mcclient.ClientSession, args *createOpt) error { + params := json.NewDict() + if len(args.KubeContext) > 0 { + params.Add(json.NewString(args.KubeContext), "kube_context") + } + params.Add(json.NewString(args.Namespace), "namespace") + params.Add(json.NewString(args.ServiceAccount), "service_account") + if args.Canary { + params.Add(json.JSONTrue, "canary_image") + } + if args.Upgrade { + params.Add(json.JSONTrue, "upgrade") + } + if len(args.Image) > 0 { + params.Add(json.NewString(args.Image), "tiller_image") + } + if args.MaxHistory > 0 { + params.Add(json.NewInt(int64(args.MaxHistory)), "history_max") + } + ret, err := k8s.Tiller.CreateInContexts(s, params, args.ClusterContext()) + if err != nil { + return err + } + printObject(ret) + return nil + }) +} diff --git a/cmd/climc/shell/networks.go b/cmd/climc/shell/networks.go index fdf011d56f..52d6900a28 100644 --- a/cmd/climc/shell/networks.go +++ b/cmd/climc/shell/networks.go @@ -45,7 +45,7 @@ func init() { ID string `help:"ID or Name of zone to update"` Name string `help:"Name of zone"` Desc string `metavar:"" help:"Description"` - ServerType string `help:"server type," choices:"baremetal|guest"` + ServerType string `help:"server type," choices:"baremetal|guest|container"` StartIp string `help:"Start ip"` EndIp string `help:"end ip"` NetMask int64 `help:"Netmask"` @@ -178,7 +178,7 @@ func init() { Gateway string `help:"Default gateway"` VlanId int64 `help:"Vlan ID" default:"1"` AllocPolicy string `help:"Address allocation policy" choices:"none|stepdown|stepup|random"` - ServerType string `help:"Server type" choices:"baremetal|guest"` + ServerType string `help:"Server type" choices:"baremetal|guest|container"` Desc string `help:"Description" metavar:"DESCRIPTION"` } R(&NetworkCreateOptions{}, "network-create", "Create a virtual network", func(s *mcclient.ClientSession, args *NetworkCreateOptions) error { diff --git a/cmd/climc/shell/servers.go b/cmd/climc/shell/servers.go index 7845391197..845c02a16c 100644 --- a/cmd/climc/shell/servers.go +++ b/cmd/climc/shell/servers.go @@ -64,6 +64,7 @@ func init() { Secgroup string `help:"Secgroup ID or Name"` AdminSecgroup string `help:"AdminSecgroup ID or Name"` Hypervisor string `help:"Show server of hypervisor" choices:"kvm|esxi|container|baremetal|aliyun"` + Manager string `help:"Show servers imported from manager"` BaseListOptions } R(&ServerListOptions{}, "server-list", "List virtual servers", func(s *mcclient.ClientSession, args *ServerListOptions) error { @@ -98,6 +99,9 @@ func init() { if len(args.Hypervisor) > 0 { params.Add(jsonutils.NewString(args.Hypervisor), "hypervisor") } + if len(args.Manager) > 0 { + params.Add(jsonutils.NewString(args.Manager), "manager") + } result, err := modules.Servers.List(s, params) if err != nil { return err @@ -139,6 +143,7 @@ func init() { Net []string `help:"Network descriptions" metavar:"NETWORK"` IsolatedDevice []string `help:"Isolated device model or ID" metavar:"ISOLATED_DEVICE"` Keypair string `help:"SSH Keypair"` + Password string `help:"Default user password"` Iso string `help:"ISO image ID" metavar:"IMAGE_ID"` Ncpu int64 `help:"#CPU cores of VM server, default 1" default:"1" metavar:""` Vga string `help:"VGA driver" choices:"std|vmware|cirrus|qxl"` @@ -192,6 +197,9 @@ func init() { if args.NoAccountInit { params.Add(jsonutils.JSONFalse, "reset_password") } + if args.Password != "" { + params.Add(jsonutils.NewString(args.Password), "password") + } if len(args.Vdi) > 0 { params.Add(jsonutils.NewString(args.Vdi), "vdi") } @@ -339,8 +347,16 @@ func init() { return nil }) - R(&ServerOpsOptions{}, "server-reset", "Reset servers", func(s *mcclient.ClientSession, args *ServerOpsOptions) error { - ret := modules.Servers.BatchPerformAction(s, args.ID, "reset", nil) + type ServerResetOptions struct { + ServerOpsOptions + Hard bool `help:"Hard reset or not; default soft"` + } + R(&ServerResetOptions{}, "server-reset", "Reset servers", func(s *mcclient.ClientSession, args *ServerResetOptions) error { + params := jsonutils.NewDict() + if args.Hard { + params.Add(jsonutils.JSONTrue, "is_hard") + } + ret := modules.Servers.BatchPerformAction(s, args.ID, "reset", params) printBatchResults(ret, modules.Servers.GetColumns(s)) return nil }) @@ -484,6 +500,7 @@ func init() { DeleteKeypair bool `help:"Remove ssh Keypairs"` Deploy []string `help:"Specify deploy files in virtual server file system"` ResetPassword bool `help:"Force reset password"` + Password string `help:"Default user password"` } R(&ServerDeployOptions{}, "server-deploy", "Deploy hostname and keypair to a stopped virtual server", func(s *mcclient.ClientSession, args *ServerDeployOptions) error { params := jsonutils.NewDict() @@ -501,6 +518,9 @@ func init() { if args.ResetPassword { params.Add(jsonutils.JSONTrue, "reset_password") } + if args.Password != "" { + params.Add(jsonutils.NewString(args.Password), "password") + } srv, e := modules.Servers.PerformAction(s, args.ID, "deploy", params) if e != nil { return e @@ -511,7 +531,7 @@ func init() { type ServerSecGroupOptions struct { ID string `help:"ID or Name of server" metavar:"Guest"` - SecGrp string `help:"ID of Security Group" metavar:"Security Group" optional:"false"` + SecGrp string `help:"ID of Security Group" metavar:"Security Group" positional:"true"` } R(&ServerSecGroupOptions{}, "server-assign-secgroup", "Assign security group to a VM", func(s *mcclient.ClientSession, args *ServerSecGroupOptions) error { diff --git a/cmd/scheduler/app/server.go b/cmd/scheduler/app/server.go index 3dc1ce9a4f..e3a8f93be8 100644 --- a/cmd/scheduler/app/server.go +++ b/cmd/scheduler/app/server.go @@ -8,10 +8,10 @@ import ( "net/http" "strconv" + "gopkg.in/gin-gonic/gin.v1" "yunion.io/x/log" "yunion.io/x/pkg/util/prometheus" "yunion.io/x/pkg/utils" - "gopkg.in/gin-gonic/gin.v1" o "yunion.io/x/onecloud/cmd/scheduler/options" "yunion.io/x/onecloud/pkg/mcclient/auth" diff --git a/pkg/appsrv/dispatcher/dispatcher.go b/pkg/appsrv/dispatcher/dispatcher.go index 1aeff85240..ea95a458f9 100644 --- a/pkg/appsrv/dispatcher/dispatcher.go +++ b/pkg/appsrv/dispatcher/dispatcher.go @@ -209,8 +209,7 @@ func performClassActionHandler(ctx context.Context, w http.ResponseWriter, r *ht if data == nil { data = body.(*jsonutils.JSONDict) } - } - if data == nil { + } else { data = jsonutils.NewDict() } results, err := manager.PerformClassAction(ctx, params[""], query, data) diff --git a/pkg/cloudcommon/cronman/cronman.go b/pkg/cloudcommon/cronman/cronman.go index b2e443d983..1dba9a33b2 100644 --- a/pkg/cloudcommon/cronman/cronman.go +++ b/pkg/cloudcommon/cronman/cronman.go @@ -1,33 +1,34 @@ package cronman import ( - "time" - "runtime/debug" "context" + "reflect" + "runtime" + "runtime/debug" + "time" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/appctx" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" - "reflect" - "runtime" ) const ( - DEFAULT_CRON_INTERVAL = 60*time.Second // default resolution is 1 monutes + DEFAULT_CRON_INTERVAL = 60 * time.Second // default resolution is 1 monutes ) type SCronJobManager struct { checkInterval time.Duration - timer *time.Timer - jobs []SCronJob + timer *time.Timer + jobs []SCronJob } type SCronJob struct { - name string + name string runInterval time.Duration - job func(ctx context.Context, userCred mcclient.TokenCredential) - lastRun time.Time + job func(ctx context.Context, userCred mcclient.TokenCredential) + lastRun time.Time } func NewCronJobManager(interval time.Duration) *SCronJobManager { @@ -89,7 +90,7 @@ func runJob(name string, job func(ctx context.Context, userCred mcclient.TokenCr }() ctx := context.Background() + ctx = context.WithValue(ctx, appctx.APP_CONTEXT_KEY_APPNAME, "Region-Corn-Service") userCred := auth.AdminCredential() job(ctx, userCred) } - diff --git a/pkg/cloudcommon/db/db_dispatcher.go b/pkg/cloudcommon/db/db_dispatcher.go index 4a661bbd77..57b6ea13c9 100644 --- a/pkg/cloudcommon/db/db_dispatcher.go +++ b/pkg/cloudcommon/db/db_dispatcher.go @@ -251,6 +251,30 @@ func applyListItemsGeneralFilters(manager IModelManager, q *sqlchemy.SQuery, return q, nil } +func applyListItemsGeneralJointFilters(manager IModelManager, q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, jointFilters []string, filterAny bool) (*sqlchemy.SQuery, error) { + for _, f := range jointFilters { + jfc := filterclause.ParseJointFilterClause(f) + if jfc != nil { + jointModelManager := GetModelManager(jfc.GetJointModelName()) + schFields := searchFields(jointModelManager, userCred) + if ok, _ := utils.InStringArray(jfc.GetField(), schFields); ok { + sq := jointModelManager.Query(jfc.ReleatedKey) + cond := jfc.GetJointFilter(sq) + if cond != nil { + sq = sq.Filter(cond) + if filterAny { + q = q.Filter(sqlchemy.OR(sqlchemy.In(q.Field("id"), sq))) + } else { + q = q.Filter(sqlchemy.AND(sqlchemy.In(q.Field("id"), sq))) + } + } + } + } + } + return q, nil +} + func listItemQueryFilters(manager IModelManager, ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { @@ -272,11 +296,15 @@ func listItemQueryFilters(manager IModelManager, ctx context.Context, q *sqlchem return nil, err } } + filterAny, _ := query.Bool("filter_any") filters := jsonutils.GetQueryStringArray(query, "filter") if len(filters) > 0 { - filterAny, _ := query.Bool("filter_any") q, err = applyListItemsGeneralFilters(manager, q, userCred, filters, filterAny) } + jointFilter := jsonutils.GetQueryStringArray(query, "joint_filter") + if len(jointFilter) > 0 { + q, _ = applyListItemsGeneralJointFilters(manager, q, userCred, jointFilter, filterAny) + } return q, nil } @@ -524,7 +552,7 @@ func (dispatcher *DBModelDispatcher) tryGetModelProperty(ctx context.Context, pr } func (dispatcher *DBModelDispatcher) Get(ctx context.Context, idStr string, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { - log.Debugf("Get %s", idStr) + // log.Debugf("Get %s", idStr) userCred := fetchUserCredential(ctx) data, err := dispatcher.tryGetModelProperty(ctx, idStr, query) @@ -541,7 +569,7 @@ func (dispatcher *DBModelDispatcher) Get(ctx context.Context, idStr string, quer } else if err != nil { return nil, err } - log.Debugf("Get found %s", model) + // log.Debugf("Get found %s", model) if !model.AllowGetDetails(ctx, userCred, query) { return nil, httperrors.NewForbiddenError("Not allow to get details") } @@ -841,7 +869,12 @@ func (dispatcher *DBModelDispatcher) PerformClassAction(ctx context.Context, act managerValue := reflect.ValueOf(dispatcher.modelManager) if action == "check-create-data" { - return dispatcher.modelManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data.(*jsonutils.JSONDict)) + manager := dispatcher.modelManager + if body, err := data.(*jsonutils.JSONDict).Get(manager.Keyword()); err != nil { + return nil, httperrors.NewGeneralError(err) + } else { + return manager.ValidateCreateData(ctx, userCred, ownerProjId, query, body.(*jsonutils.JSONDict)) + } } return objectPerformAction(dispatcher, managerValue, ctx, userCred, action, query, data) } @@ -975,7 +1008,6 @@ func updateItem(manager IModelManager, item IModel, ctx context.Context, userCre return nil, httperrors.NewGeneralError(err) } item.PreUpdate(ctx, userCred, query, dataDict) - diff, err := manager.TableSpec().Update(item, func() error { filterData := dataDict.CopyIncludes(updateFields(manager, userCred)...) err = filterData.Unmarshal(item) diff --git a/pkg/cloudcommon/db/taskman/tasks.go b/pkg/cloudcommon/db/taskman/tasks.go index bd8578ca36..f221ae6f11 100644 --- a/pkg/cloudcommon/db/taskman/tasks.go +++ b/pkg/cloudcommon/db/taskman/tasks.go @@ -246,17 +246,21 @@ func (manager *STaskManager) NewParallelTask(ctx context.Context, taskName strin } func (manager *STaskManager) fetchTask(idStr string) *STask { - task, err := db.NewModelObject(manager) + iTask, err := db.NewModelObject(manager) if err != nil { log.Errorf("New task object fail: %s", err) return nil } - err = manager.Query().Equals("id", idStr).First(task) + err = manager.Query().Equals("id", idStr).First(iTask) if err != nil { log.Errorf("GetTask %s fail: %s", idStr, err) return nil } - return task.(*STask) + task := iTask.(*STask) + if task.Params == nil { + task.Params = jsonutils.NewDict() + } + return task } func (manager *STaskManager) execTask(taskId string, data jsonutils.JSONObject) { @@ -285,20 +289,21 @@ func (manager *STaskManager) execTask(taskId string, data jsonutils.JSONObject) } } -func execITask(taskValue reflect.Value, task *STask, data jsonutils.JSONObject, isMulti bool) { +func execITask(taskValue reflect.Value, task *STask, odata jsonutils.JSONObject, isMulti bool) { var err error ctxData := task.GetRequestContext() ctx := ctxData.GetContext() taskFailed := false + data := odata if data != nil { taskStatus, _ := data.GetString("__status__") if len(taskStatus) > 0 && taskStatus != "OK" { taskFailed = true - data, err = data.Get("reason") + data, err = data.Get("__reason__") if err != nil { - data = jsonutils.NewString("Task failed due to unknown remote errors!") + data = jsonutils.NewString(fmt.Sprintf("Task failed due to unknown remote errors! %s", odata)) } } } else { @@ -324,7 +329,13 @@ func execITask(taskValue reflect.Value, task *STask, data jsonutils.JSONObject, if !funcValue.IsValid() || funcValue.IsNil() { msg := fmt.Sprintf("Stage %s not found", stageName) - log.Errorf(msg) + if taskFailed { + // failed handler is optional, ignore the error + log.Warningf(msg) + msg, _ = data.GetString() + } else { + log.Errorf(msg) + } task.SetStageFailed(ctx, msg) task.SaveRequestContext(&ctxData) return @@ -412,9 +423,11 @@ func (self *STask) GetParentTask() *STask { func (self *STask) GetRequestContext() appctx.AppContextData { ctxData := appctx.AppContextData{} - ctxJson, _ := self.Params.Get(REQUEST_CONTEXT_KEY) - if ctxJson != nil { - ctxJson.Unmarshal(&ctxData) + if self.Params != nil { + ctxJson, _ := self.Params.Get(REQUEST_CONTEXT_KEY) + if ctxJson != nil { + ctxJson.Unmarshal(&ctxData) + } } return ctxData } @@ -542,7 +555,7 @@ func (self *STask) NotifyParentTaskFailure(ctx context.Context, reason string) { if len(reason) > 100 { reason = reason[:100] + "..." } - body.Add(jsonutils.NewString(fmt.Sprintf("Subtask %s failed: %s", self.TaskName, reason))) + body.Add(jsonutils.NewString(fmt.Sprintf("Subtask %s failed: %s", self.TaskName, reason)), "__reason__") self.NotifyParentTaskComplete(ctx, body, true) } diff --git a/pkg/cloudcommon/db/virtualresource.go b/pkg/cloudcommon/db/virtualresource.go index db56d599f4..8220398d3c 100644 --- a/pkg/cloudcommon/db/virtualresource.go +++ b/pkg/cloudcommon/db/virtualresource.go @@ -161,16 +161,16 @@ func (model *SVirtualResourceBase) AllowPerformMetadata(ctx context.Context, use } func (model *SVirtualResourceBase) GetTenantCache(ctx context.Context) (*STenant, error) { - log.Debugf("Get tenant by Id %s", model.ProjectId) + // log.Debugf("Get tenant by Id %s", model.ProjectId) return TenantCacheManager.FetchTenantById(ctx, model.ProjectId) } func (model *SVirtualResourceBase) getMoreDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, extra *jsonutils.JSONDict) *jsonutils.JSONDict { if userCred.IsSystemAdmin() { - log.Debugf("GetCustomizeColumns") + // log.Debugf("GetCustomizeColumns") tobj, err := model.GetTenantCache(ctx) if err == nil { - log.Debugf("GetTenantFromCache %s", jsonutils.Marshal(tobj)) + // log.Debugf("GetTenantFromCache %s", jsonutils.Marshal(tobj)) extra.Add(jsonutils.NewString(tobj.GetName()), "tenant") } else { log.Errorf("GetTenantCache fail %s", err) diff --git a/pkg/cloudprovider/resources.go b/pkg/cloudprovider/resources.go index f388d66feb..ee345facf1 100644 --- a/pkg/cloudprovider/resources.go +++ b/pkg/cloudprovider/resources.go @@ -18,6 +18,7 @@ type ICloudResource interface { Refresh() error IsEmulated() bool + GetMetadata() *jsonutils.JSONDict } type ICloudRegion interface { @@ -153,7 +154,13 @@ type ICloudVM interface { StopVM(isForce bool) error DeleteVM() error + UpdateVM(name string) error + RebuildRoot(imageId string) error + DeployVM(name string, password string, publicKey string, resetPassword bool, deleteKeypair bool, description string) error + ChangeConfig(instanceId string, ncpu int, vmem int) error GetVNCInfo() (jsonutils.JSONObject, error) + AttachDisk(diskId string) error + DetachDisk(diskId string) error } type ICloudNic interface { @@ -193,7 +200,8 @@ type ICloudDisk interface { GetCacheMode() string GetMountpoint() string Delete() error - Resize(int64) error + + Resize(newSize int64) error } type ICloudVpc interface { diff --git a/pkg/compute/guestdrivers/aliyun.go b/pkg/compute/guestdrivers/aliyun.go index a58b046b80..42eeaa6eed 100644 --- a/pkg/compute/guestdrivers/aliyun.go +++ b/pkg/compute/guestdrivers/aliyun.go @@ -104,7 +104,6 @@ func (self *SAliyunGuestDriver) GetJsonDescAtHost(ctx context.Context, guest *mo imageId := disk.GetTemplateId() scimg := models.StoragecachedimageManager.GetStoragecachedimage(cache.Id, imageId) config.ExternalImageId = scimg.ExternalId - img := scimg.GetCachedimage() config.OsDistribution, _ = img.Info.GetString("properties", "os_distribution") config.OsVersion, _ = img.Info.GetString("properties", "os_version") @@ -119,108 +118,159 @@ func (self *SAliyunGuestDriver) GetJsonDescAtHost(ctx context.Context, guest *mo } type SDiskInfo struct { - Size int - Uuid string + Size int + Uuid string + Metadata map[string]string } func (self *SAliyunGuestDriver) RequestDeployGuestOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error { config := guest.GetDeployConfigOnHost(ctx, host, task.GetParams()) - onfinish, err := config.GetString("on_finish") + /* onfinish, err := config.GetString("on_finish") if err != nil { return err - } + } */ + action, err := config.GetString("action") if err != nil { return err } - if action != "create" { - return fmt.Errorf("Action %s not supported", action) - } - ihost, err := host.GetIHost() if err != nil { return err } - desc := SAliyunVMCreateConfig{} - err = config.Unmarshal(&desc, "desc") - if err != nil { - return err - } - - taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - passwd := seclib2.RandomPassword2(12) - - iVM, err := ihost.CreateVM(desc.Name, desc.ExternalImageId, desc.SysDiskSize, desc.Cpu, desc.Memory, desc.ExternalNetworkId, - desc.IpAddr, desc.Description, passwd, desc.StorageType, desc.DataDisks, desc.PublicKey) + if action == "create" { + desc := SAliyunVMCreateConfig{} + err = config.Unmarshal(&desc, "desc") if err != nil { - return nil, err - } - log.Debugf("VMcreated %s, wait status ready ...", iVM.GetGlobalId()) - err = cloudprovider.WaitStatus(iVM, models.VM_READY, time.Second*5, time.Second*1800) - if err != nil { - return nil, err - } - log.Debugf("VMcreated %s, and status is ready", iVM.GetGlobalId()) - - iVM, err = ihost.GetIVMById(iVM.GetGlobalId()) - if err != nil { - log.Errorf("cannot find vm %s", err) - return nil, err + return err } - if len(guest.SecgrpId) > 0 { - if err := iVM.SyncSecurityGroup(guest.SecgrpId, guest.GetSecgroupName(), guest.GetSecRules()); err != nil { - log.Errorf("SyncSecurityGroup error: %v", err) - return nil, err - } - } + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + passwd := seclib2.RandomPassword2(12) - if onfinish == "none" { - err = iVM.StartVM() + iVM, err := ihost.CreateVM(desc.Name, desc.ExternalImageId, desc.SysDiskSize, desc.Cpu, desc.Memory, desc.ExternalNetworkId, + desc.IpAddr, desc.Description, passwd, desc.StorageType, desc.DataDisks, desc.PublicKey) if err != nil { return nil, err } - } - - encpasswd, err := utils.EncryptAESBase64(guest.Id, passwd) - if err != nil { - log.Errorf("encrypt password failed %s", err) - } - - data := jsonutils.NewDict() - data.Add(jsonutils.NewString(iVM.GetOSType()), "os") - data.Add(jsonutils.NewString("root"), "account") - data.Add(jsonutils.NewString(encpasswd), "key") - - if len(desc.OsDistribution) > 0 { - data.Add(jsonutils.NewString(desc.OsDistribution), "distro") - } - if len(desc.OsVersion) > 0 { - data.Add(jsonutils.NewString(desc.OsVersion), "version") - } - - idisks, err := iVM.GetIDisks() - - if err != nil { - log.Errorf("GetiDisks error %s", err) - } else { - diskInfo := make([]SDiskInfo, len(idisks)) - for i := 0; i < len(idisks); i += 1 { - dinfo := SDiskInfo{} - dinfo.Uuid = idisks[i].GetGlobalId() - dinfo.Size = idisks[i].GetDiskSizeMB() - diskInfo[i] = dinfo + log.Debugf("VMcreated %s, wait status ready ...", iVM.GetGlobalId()) + err = cloudprovider.WaitStatus(iVM, models.VM_READY, time.Second*5, time.Second*1800) + if err != nil { + return nil, err } - data.Add(jsonutils.Marshal(&diskInfo), "disks") + log.Debugf("VMcreated %s, and status is ready", iVM.GetGlobalId()) + + iVM, err = ihost.GetIVMById(iVM.GetGlobalId()) + if err != nil { + log.Errorf("cannot find vm %s", err) + return nil, err + } + + if len(guest.SecgrpId) > 0 { + if err := iVM.SyncSecurityGroup(guest.SecgrpId, guest.GetSecgroupName(), guest.GetSecRules()); err != nil { + log.Errorf("SyncSecurityGroup error: %v", err) + return nil, err + } + } + + /*if onfinish == "none" { + err = iVM.StartVM() + if err != nil { + return nil, err + } + }*/ + + encpasswd, err := utils.EncryptAESBase64(guest.Id, passwd) + if err != nil { + log.Errorf("encrypt password failed %s", err) + } + + data := jsonutils.NewDict() + data.Add(jsonutils.NewString(iVM.GetOSType()), "os") + data.Add(jsonutils.NewString("root"), "account") + data.Add(jsonutils.NewString(encpasswd), "key") + + if len(desc.OsDistribution) > 0 { + data.Add(jsonutils.NewString(desc.OsDistribution), "distro") + } + if len(desc.OsVersion) > 0 { + data.Add(jsonutils.NewString(desc.OsVersion), "version") + } + + idisks, err := iVM.GetIDisks() + + if err != nil { + log.Errorf("GetiDisks error %s", err) + } else { + diskInfo := make([]SDiskInfo, len(idisks)) + for i := 0; i < len(idisks); i += 1 { + dinfo := SDiskInfo{} + dinfo.Uuid = idisks[i].GetGlobalId() + dinfo.Size = idisks[i].GetDiskSizeMB() + if metaData := idisks[i].GetMetadata(); metaData != nil { + dinfo.Metadata = make(map[string]string, 0) + if err := metaData.Unmarshal(dinfo.Metadata); err != nil { + log.Errorf("Get disk %s metadata info error: %v", idisks[i].GetName(), err) + } + } + diskInfo[i] = dinfo + } + data.Add(jsonutils.Marshal(&diskInfo), "disks") + } + + data.Add(jsonutils.NewString(iVM.GetGlobalId()), "uuid") + data.Add(iVM.GetMetadata(), "metadata") + + return data, nil + }) + } else if action == "deploy" { + iVM, err := ihost.GetIVMById(guest.GetExternalId()) + if err != nil || iVM == nil { + log.Errorf("cannot find vm %s", err) + return fmt.Errorf("cannot find vm") } - data.Add(jsonutils.NewString(iVM.GetGlobalId()), "uuid") + params := task.GetParams() + log.Debugf("Deploy VM params %s", params.String()) + var name string + if v, e := params.GetString("name"); e != nil { + name = v + } + var description string + if v, e := params.GetString("description"); e != nil { + description = v + } + resetPassword := jsonutils.QueryBoolean(params, "reset_password", false) + deleteKeypair := jsonutils.QueryBoolean(params, "__delete_keypair__", false) + password, _ := params.GetString("password") + if resetPassword && len(password) == 0 { + password = seclib2.RandomPassword2(12) + } + + publicKey := "" + if k, e := config.GetString("public_key"); e != nil { + publicKey = k + } + + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + encpasswd, err := utils.EncryptAESBase64(guest.Id, password) + if err != nil { + log.Errorf("encrypt password failed %s", err) + } + + data := jsonutils.NewDict() + data.Add(jsonutils.NewString("root"), "account") // 用户名 + data.Add(jsonutils.NewString(encpasswd), "key") // 密码 + e := iVM.DeployVM(name, password, publicKey, resetPassword, deleteKeypair, description) + return data, e + }) + } else { + return fmt.Errorf("Action %s not supported", action) + } - return data, nil - }) return nil } @@ -244,6 +294,13 @@ func (self *SAliyunGuestDriver) OnGuestDeployTaskDataReceived(ctx context.Contex disk.DiskSize = diskInfo[i].Size disk.ExternalId = diskInfo[i].Uuid disk.Status = models.DISK_READY + if len(diskInfo[i].Metadata) > 0 { + for key, value := range diskInfo[i].Metadata { + if err := disk.SetMetadata(ctx, key, value, task.GetUserCred()); err != nil { + log.Errorf("set disk %s mata %s => %s error: %v", disk.Name, key, value, err) + } + } + } return nil }) if err != nil { @@ -259,6 +316,20 @@ func (self *SAliyunGuestDriver) OnGuestDeployTaskDataReceived(ctx context.Contex if len(uuid) > 0 { guest.SetExternalId(uuid) } + + if metaData, _ := data.Get("metadata"); metaData != nil { + meta := make(map[string]string, 0) + if err := metaData.Unmarshal(meta); err != nil { + log.Errorf("Get guest %s metadata error: %v", guest.Name, err) + } else { + for key, value := range meta { + if err := guest.SetMetadata(ctx, key, value, task.GetUserCred()); err != nil { + log.Errorf("set guest %s mata %s => %s error: %v", guest.Name, key, value, err) + } + } + } + } + guest.SaveDeployInfo(ctx, task.GetUserCred(), data) return nil } @@ -275,7 +346,6 @@ func (self *SAliyunGuestDriver) RequestSyncConfigOnHost(ctx context.Context, gue return nil, err } } else { - iVM.Refresh() if iDisks, err := iVM.GetIDisks(); err != nil { return nil, err } else { @@ -293,18 +363,16 @@ func (self *SAliyunGuestDriver) RequestSyncConfigOnHost(ctx context.Context, gue if err := compare.CompareSets(disks, iDisks, &added, &commondb, &commonext, &removed); err != nil { return nil, err } - // for _, disk := range removed { - // log.Errorf("need remove %v", disk) - // if err := iVM.DetachDisk(disk.GetId()); err != nil { - // return nil, err - // } - // } - // for _, disk := range added { - // log.Errorf("need add %v", disk) - // if err := iVM.AttachDisk(disk.ExternalId); err != nil { - // return nil, err - // } - // } + for _, disk := range removed { + if err := iVM.DetachDisk(disk.GetId()); err != nil { + return nil, err + } + } + for _, disk := range added { + if err := iVM.AttachDisk(disk.ExternalId); err != nil { + return nil, err + } + } } } } @@ -312,3 +380,118 @@ func (self *SAliyunGuestDriver) RequestSyncConfigOnHost(ctx context.Context, gue }) return nil } + +type SAliyunVMChangeConfig struct { + InstanceId string + Cpu int + Memory int +} + +func (self *SAliyunGuestDriver) DoGuestCreateDisksTask(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + subtask, err := taskman.TaskManager.NewTask(ctx, "AliyunGuestCreateDiskTask", guest, task.GetUserCred(), task.GetParams(), task.GetTaskId(), "", nil) + if err != nil { + return err + } + subtask.ScheduleRun(nil) + return nil +} + +func (self *SAliyunGuestDriver) AllowReconfigGuest() bool { + return true +} + +func (self *SAliyunGuestDriver) RequestChangeVmConfig(ctx context.Context, guest *models.SGuest, task taskman.ITask, vcpuCount, vmemSize int64) error { + config := SAliyunVMChangeConfig{} + config.InstanceId = guest.GetExternalId() + config.Cpu = int(vcpuCount) + config.Memory = int(vmemSize) + ihost, err := guest.GetHost().GetIHost() + if err != nil { + return err + } + + iVM, err := ihost.GetIVMById(config.InstanceId) + if err != nil { + return err + } + + if int(guest.VcpuCount) != config.Cpu || guest.VmemSize != config.Memory { + err = iVM.ChangeConfig(config.InstanceId, config.Cpu, config.Memory) + if err != nil { + return err + } + } + + log.Debugf("VMchangeConfig %s, wait status ready ...", iVM.GetGlobalId()) + err = cloudprovider.WaitStatus(iVM, models.VM_READY, time.Second*5, time.Second*300) + if err != nil { + return err + } + log.Debugf("VMchangeConfig %s, and status is ready", iVM.GetGlobalId()) + return nil +} + +func (self *SAliyunGuestDriver) RequestStartOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, userCred mcclient.TokenCredential, task taskman.ITask) (jsonutils.JSONObject, error) { + ihost, e := host.GetIHost() + if e != nil { + return nil, e + } + + ivm, e := ihost.GetIVMById(guest.GetExternalId()) + if e != nil { + return nil, e + } + + err := ivm.StartVM() + if err != nil { + return nil, e + } + + result := jsonutils.NewDict() + result.Add(jsonutils.NewBool(true), "is_running") + return result, e +} + +func (self *SAliyunGuestDriver) RequestRebuildRootDisk(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + ihost, e := guest.GetHost().GetIHost() + if e != nil { + return e + } + + externalId := guest.GetExternalId() + if len(externalId) <= 0 { + return fmt.Errorf("external id not found") + } + + disks := guest.GetDisks() + if len(disks) <= 0 { + return fmt.Errorf("guest has no disk") + } + + imageId := guest.CategorizeDisks().Root.TemplateId + cacheId := disks[0].GetDisk().GetStorage().GetStoragecache().Id + externalImageId := models.StoragecachedimageManager.GetStoragecachedimage(cacheId, imageId).ExternalId + if len(externalImageId) <= 0 { + return fmt.Errorf("external image (%s) id is not found", imageId) + } + + iVM, err := ihost.GetIVMById(externalId) + if err != nil { + return err + } + + err = iVM.RebuildRoot(externalImageId) + if err != nil { + return err + } + + log.Debugf("VMrebuildRoot %s, wait status ready ...", iVM.GetGlobalId()) + err = cloudprovider.WaitStatus(iVM, models.VM_READY, time.Second*5, time.Second*1800) + if err != nil { + return err + } + log.Debugf("VMrebuildRoot %s, and status is ready", iVM.GetGlobalId()) + + task.ScheduleRun(nil) + return nil +} diff --git a/pkg/compute/guestdrivers/base.go b/pkg/compute/guestdrivers/base.go index b73a757637..a63aa1470c 100644 --- a/pkg/compute/guestdrivers/base.go +++ b/pkg/compute/guestdrivers/base.go @@ -102,6 +102,14 @@ func (self *SBaseGuestDriver) RqeuestSuspendOnHost(ctx context.Context, guest *m return fmt.Errorf("Not Implement") } +func (self *SBaseGuestDriver) StartGuestResetTask(guest *models.SGuest, ctx context.Context, userCred mcclient.TokenCredential, isHard bool, parentTaskId string) error { + return fmt.Errorf("Not Implement") +} + +func (self *SBaseGuestDriver) RequestSoftReset(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + return fmt.Errorf("Not Implement") +} + func (self *SBaseGuestDriver) AllowReconfigGuest() bool { return true } diff --git a/pkg/compute/guestdrivers/kvm.go b/pkg/compute/guestdrivers/kvm.go index 2a04d43eda..0c3e1ab525 100644 --- a/pkg/compute/guestdrivers/kvm.go +++ b/pkg/compute/guestdrivers/kvm.go @@ -206,6 +206,11 @@ func (self *SKVMGuestDriver) RequestChangeVmConfig(ctx context.Context, guest *m return nil } +func (self *SKVMGuestDriver) RequestSoftReset(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + _, err := guest.SendMonitorCommand(ctx, task.GetUserCred(), "system_reset") + return err +} + func (self *SKVMGuestDriver) RequestDetachDisk(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { return guest.StartSyncTask(ctx, task.GetUserCred(), false, task.GetTaskId()) } @@ -219,18 +224,21 @@ func (self *SKVMGuestDriver) RequestDeleteDetachedDisk(ctx context.Context, disk } func (self *SKVMGuestDriver) RequestSyncConfigOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error { - desc := guest.GetDriver().GetJsonDescAtHost(ctx, guest, host) - body := jsonutils.NewDict() - body.Add(desc, "desc") - if fw_only, _ := task.GetParams().Bool("fw_only"); fw_only { - body.Add(jsonutils.JSONTrue, "fw_only") + if guest.Status == models.VM_RUNNING { + desc := guest.GetDriver().GetJsonDescAtHost(ctx, guest, host) + body := jsonutils.NewDict() + body.Add(desc, "desc") + if fw_only, _ := task.GetParams().Bool("fw_only"); fw_only { + body.Add(jsonutils.JSONTrue, "fw_only") + } + url := fmt.Sprintf("/servers/%s/sync", guest.Id) + header := http.Header{} + header.Add("X-Task-Id", task.GetTaskId()) + header.Add("X-Region-Version", "v2") + _, err := host.Request(task.GetUserCred(), "POST", url, header, body) + return err } - url := fmt.Sprintf("/servers/%s/sync", guest.Id) - header := http.Header{} - header.Add("X-Task-Id", task.GetTaskId()) - header.Add("X-Region-Version", "v2") - _, err := host.Request(task.GetUserCred(), "POST", url, header, body) - return err + return nil } func (self *SKVMGuestDriver) RqeuestSuspendOnHost(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { diff --git a/pkg/compute/guestdrivers/virtualization.go b/pkg/compute/guestdrivers/virtualization.go index c6311bd5f3..7efdd6d475 100644 --- a/pkg/compute/guestdrivers/virtualization.go +++ b/pkg/compute/guestdrivers/virtualization.go @@ -95,6 +95,19 @@ func (self *SVirtualizedGuestDriver) StartGuestStopTask(guest *models.SGuest, ct return nil } +func (self *SVirtualizedGuestDriver) StartGuestResetTask(guest *models.SGuest, ctx context.Context, userCred mcclient.TokenCredential, isHard bool, parentTaskId string) error { + var taskName = "GuestSoftResetTask" + if isHard { + taskName = "GuestHardResetTask" + } + task, err := taskman.TaskManager.NewTask(ctx, taskName, guest, userCred, nil, parentTaskId, "", nil) + if err != nil { + return err + } + task.ScheduleRun(nil) + return nil +} + func (self *SVirtualizedGuestDriver) OnGuestDeployTaskComplete(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { if jsonutils.QueryBoolean(task.GetParams(), "restart", false) { task.SetStage("OnDeployStartGuestComplete", nil) diff --git a/pkg/compute/hostdrivers/aliyun.go b/pkg/compute/hostdrivers/aliyun.go index 0bcfa18005..4c1d7bf029 100644 --- a/pkg/compute/hostdrivers/aliyun.go +++ b/pkg/compute/hostdrivers/aliyun.go @@ -48,7 +48,7 @@ func (self *SAliyunHostDriver) CheckAndSetCacheImage(ctx context.Context, host * return nil } -func (self *SAliyunHostDriver) RequestAllocateDiskOnStorage(host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask, content *jsonutils.JSONDict) error { +func (self *SAliyunHostDriver) RequestAllocateDiskOnStorage(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask, content *jsonutils.JSONDict) error { if iCloudStorage, err := storage.GetIStorage(); err != nil { return err } else { @@ -61,6 +61,20 @@ func (self *SAliyunHostDriver) RequestAllocateDiskOnStorage(host *models.SHost, } else { if _, err := disk.GetModelManager().TableSpec().Update(disk, func() error { disk.ExternalId = iDisk.GetGlobalId() + + if metaData := iDisk.GetMetadata(); metaData != nil { + meta := make(map[string]string) + if err := metaData.Unmarshal(meta); err != nil { + log.Errorf("Get disk %s Metadata error: %v", disk.Name, err) + } else { + for key, value := range meta { + if err := disk.SetMetadata(ctx, key, value, task.GetUserCred()); err != nil { + log.Errorf("set disk %s mata %s => %s error: %v", disk.Name, key, value, err) + } + } + } + } + return nil }); err != nil { log.Errorf("Update disk externalId err: %v", err) @@ -81,9 +95,13 @@ func (self *SAliyunHostDriver) RequestDeallocateDiskOnHost(host *models.SHost, s return err } else if iDisk, err := iCloudStorage.GetIDisk(disk.GetExternalId()); err != nil { return err + } else if err := iDisk.Delete(); err != nil { + return err } else { - return iDisk.Delete() + data := jsonutils.NewDict() + task.ScheduleRun(data) } + return nil } func (self *SAliyunHostDriver) RequestResizeDiskOnHostOnline(host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { diff --git a/pkg/compute/hostdrivers/kvm.go b/pkg/compute/hostdrivers/kvm.go index 0a5cbca5b2..ed08542a36 100644 --- a/pkg/compute/hostdrivers/kvm.go +++ b/pkg/compute/hostdrivers/kvm.go @@ -74,7 +74,7 @@ func (self *SKVMHostDriver) CheckAndSetCacheImage(ctx context.Context, host *mod return nil } -func (self *SKVMHostDriver) RequestAllocateDiskOnStorage(host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask, content *jsonutils.JSONDict) error { +func (self *SKVMHostDriver) RequestAllocateDiskOnStorage(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask, content *jsonutils.JSONDict) error { header := http.Header{} header.Add("X-Task-Id", task.GetTaskId()) header.Add("X-Region-Version", "v2") diff --git a/pkg/compute/models/baremetalstatus.go b/pkg/compute/models/baremetalstatus.go new file mode 100644 index 0000000000..5c4561f14c --- /dev/null +++ b/pkg/compute/models/baremetalstatus.go @@ -0,0 +1,22 @@ +package models + +const ( + BAREMETAL_INIT = "init" + BAREMETAL_PREPARE = "prepare" + BAREMETAL_PREPARE_FAIL = "prepare_fail" + BAREMETAL_READY = "ready" + BAREMETAL_RUNNING = "running" + BAREMETAL_MAINTAINING = "maintaining" + BAREMETAL_START_MAINTAIN = "start_maintain" + BAREMETAL_DELETING = "deleting" + BAREMETAL_DELETE = "delete" + BAREMETAL_DELETE_FAIL = "delete_fail" + BAREMETAL_UNKNOWN = "unknown" + BAREMETAL_SYNCING_STATUS = "syncing_status" + BAREMETAL_SYNC = "sync" + BAREMETAL_SYNC_FAIL = "sync_fail" + BAREMETAL_START_CONVERT = "start_convert" + BAREMETAL_CONVERTING = "converting" + BAREMETAL_START_FAIL = "start_fail" + BAREMETAL_STOP_FAIL = "stop_fail" +) diff --git a/pkg/compute/models/cloudregions.go b/pkg/compute/models/cloudregions.go index e3c31efbba..44a1901655 100644 --- a/pkg/compute/models/cloudregions.go +++ b/pkg/compute/models/cloudregions.go @@ -145,9 +145,9 @@ func (manager *SCloudregionManager) SyncRegions(ctx context.Context, userCred mc _, err = removed[i].PerformDisable(ctx, userCred, nil, nil) } if err != nil { - syncResult.Delete() - } else { syncResult.DeleteError(err) + } else { + syncResult.Delete() } } } diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index 35aba24217..b05fa85b5e 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -16,6 +16,7 @@ import ( "yunion.io/x/pkg/util/osprofile" "yunion.io/x/pkg/util/regutils" "yunion.io/x/pkg/util/sysutils" + "yunion.io/x/pkg/util/timeutils" "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" @@ -296,7 +297,7 @@ func (self *SDisk) StartDiskCreateTask(ctx context.Context, userCred mcclient.To return nil } -func (self *SDisk) StartAllocate(host *SHost, storage *SStorage, taskId string, userCred mcclient.TokenCredential, rebuild bool, snapshot string, task taskman.ITask) error { +func (self *SDisk) StartAllocate(ctx context.Context, host *SHost, storage *SStorage, taskId string, userCred mcclient.TokenCredential, rebuild bool, snapshot string, task taskman.ITask) error { log.Infof("Allocating disk on host %s ...", host.GetName()) templateId := self.GetTemplateId() @@ -325,7 +326,7 @@ func (self *SDisk) StartAllocate(host *SHost, storage *SStorage, taskId string, if rebuild { content.Add(jsonutils.JSONTrue, "rebuild") } - return host.GetHostDriver().RequestAllocateDiskOnStorage(host, storage, self, task, content) + return host.GetHostDriver().RequestAllocateDiskOnStorage(ctx, host, storage, self, task, content) } func (self *SDisk) AllowPerformResize(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { @@ -384,6 +385,13 @@ func (self *SDisk) GetStorage() *SStorage { return nil } +func (self *SDisk) GetCloudprovider() *SCloudprovider { + if storage := self.GetStorage(); storage != nil { + return storage.GetCloudprovider() + } + return nil +} + func (self *SDisk) GetPathAtHost(host *SHost) string { storage := self.GetStorage() if storage.StorageType == STORAGE_RBD { @@ -425,7 +433,7 @@ func (manager *SDiskManager) getDisksByStorage(storage *SStorage) ([]SDisk, erro return disks, nil } -func (manager *SDiskManager) syncCloudDisk(userCred mcclient.TokenCredential, vdisk cloudprovider.ICloudDisk) (*SDisk, error) { +func (manager *SDiskManager) syncCloudDisk(ctx context.Context, userCred mcclient.TokenCredential, vdisk cloudprovider.ICloudDisk) (*SDisk, error) { diskObj, err := manager.FetchByExternalId(vdisk.GetGlobalId()) if err != nil { if err == sql.ErrNoRows { @@ -436,13 +444,13 @@ func (manager *SDiskManager) syncCloudDisk(userCred mcclient.TokenCredential, vd return nil, err } storage := storageObj.(*SStorage) - return manager.newFromCloudDisk(userCred, vdisk, storage) + return manager.newFromCloudDisk(ctx, userCred, vdisk, storage) } else { return nil, err } } else { disk := diskObj.(*SDisk) - err = disk.syncWithCloudDisk(userCred, vdisk) + err = disk.syncWithCloudDisk(ctx, userCred, vdisk) if err != nil { return nil, err } @@ -482,7 +490,7 @@ func (manager *SDiskManager) SyncDisks(ctx context.Context, userCred mcclient.To } for i := 0; i < len(commondb); i += 1 { - err = commondb[i].syncWithCloudDisk(userCred, commonext[i]) + err = commondb[i].syncWithCloudDisk(ctx, userCred, commonext[i]) if err != nil { syncResult.UpdateError(err) } else { @@ -493,7 +501,7 @@ func (manager *SDiskManager) SyncDisks(ctx context.Context, userCred mcclient.To } for i := 0; i < len(added); i += 1 { - new, err := manager.newFromCloudDisk(userCred, added[i], storage) + new, err := manager.newFromCloudDisk(ctx, userCred, added[i], storage) if err != nil { syncResult.AddError(err) } else { @@ -506,7 +514,7 @@ func (manager *SDiskManager) SyncDisks(ctx context.Context, userCred mcclient.To return localDisks, remoteDisks, syncResult } -func (self *SDisk) syncWithCloudDisk(userCred mcclient.TokenCredential, extDisk cloudprovider.ICloudDisk) error { +func (self *SDisk) syncWithCloudDisk(ctx context.Context, userCred mcclient.TokenCredential, extDisk cloudprovider.ICloudDisk) error { _, err := self.GetModelManager().TableSpec().Update(self, func() error { extDisk.Refresh() self.Name = extDisk.GetName() @@ -527,11 +535,26 @@ func (self *SDisk) syncWithCloudDisk(userCred mcclient.TokenCredential, extDisk }) if err != nil { log.Errorf("syncWithCloudDisk error %s", err) + return err } - return err + + if metaData := extDisk.GetMetadata(); metaData != nil { + meta := make(map[string]string, 0) + if err := metaData.Unmarshal(meta); err != nil { + log.Errorf("Get VM Metadata error: %v", err) + } else { + for key, value := range meta { + if err := self.SetMetadata(ctx, key, value, userCred); err != nil { + log.Errorf("set disk %s mata %s => %s error: %v", self.Name, key, value, err) + } + } + } + } + + return nil } -func (manager *SDiskManager) newFromCloudDisk(userCred mcclient.TokenCredential, extDisk cloudprovider.ICloudDisk, storage *SStorage) (*SDisk, error) { +func (manager *SDiskManager) newFromCloudDisk(ctx context.Context, userCred mcclient.TokenCredential, extDisk cloudprovider.ICloudDisk, storage *SStorage) (*SDisk, error) { disk := SDisk{} disk.SetModelManager(manager) @@ -554,6 +577,20 @@ func (manager *SDiskManager) newFromCloudDisk(userCred mcclient.TokenCredential, log.Errorf("newFromCloudZone fail %s", err) return nil, err } + + if metaData := extDisk.GetMetadata(); metaData != nil { + meta := make(map[string]string) + if err := metaData.Unmarshal(meta); err != nil { + log.Errorf("Get VM Metadata error: %v", err) + } else { + for key, value := range meta { + if err := disk.SetMetadata(ctx, key, value, userCred); err != nil { + log.Errorf("set disk %s mata %s => %s error: %v", disk.Name, key, value, err) + } + } + } + } + return &disk, nil } @@ -693,6 +730,27 @@ func parseIsoInfo(ctx context.Context, userCred mcclient.TokenCredential, info s return image.Id, nil } +// def get_disk_spec_v2(conf): +// def _get_spec(storages): +// spec = {} +// for adapter, ss in group_by_adapter(storages).items(): +// if len(ss) == 0: +// continue +// spec[adapter] = get_disk_spec(ss) +// return spec + +// spec = {} +// for driver in DISK_DRIVERS: +// storages = [s for s in conf if s['driver'] == driver] +// if len(storages) != 0: +// spec[driver] = _get_spec(storages) +// return spec + +func GetDiskSpecV2(storageInfo jsonutils.JSONObject) *jsonutils.JSONDict { + // ToDo + return nil +} + func (self *SDisk) fetchDiskInfo(diskConfig *SDiskConfig) { if len(diskConfig.ImageId) > 0 { self.TemplateId = diskConfig.ImageId @@ -746,6 +804,49 @@ func (self *SDisk) CustomizeDelete(ctx context.Context, userCred mcclient.TokenC return self.StartDiskDeleteTask(ctx, userCred, "", false) } +func (self *SDisk) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict { + if cloudprovider := self.GetCloudprovider(); cloudprovider != nil { + extra.Add(jsonutils.NewString(cloudprovider.Provider), "provider") + } + if storage := self.GetStorage(); storage != nil { + extra.Add(jsonutils.NewString(storage.GetName()), "storage") + extra.Add(jsonutils.NewString(storage.StorageType), "storage_type") + extra.Add(jsonutils.NewString(storage.MediumType), "medium_type") + extra.Add(jsonutils.NewString(storage.ZoneId), "zone_id") + if zone := storage.getZone(); zone != nil { + extra.Add(jsonutils.NewString(zone.Name), "zone") + extra.Add(jsonutils.NewString(zone.CloudregionId), "region_id") + if region := zone.GetRegion(); region != nil { + extra.Add(jsonutils.NewString(region.Name), "region") + } + } + } + guests, guest_status := []string{}, []string{} + for _, guest := range self.GetGuests() { + guests = append(guests, guest.Name) + guest_status = append(guest_status, guest.Status) + } + extra.Add(jsonutils.NewString(strings.Join(guests, ",")), "guest") + extra.Add(jsonutils.NewInt(int64(len(guests))), "guest_count") + extra.Add(jsonutils.NewString(strings.Join(guest_status, ",")), "guest_status") + + if self.PendingDeleted { + pendingDeletedAt := self.PendingDeletedAt.Add(time.Second * time.Duration(options.Options.PendingDeleteExpireSeconds)) + extra.Add(jsonutils.NewString(timeutils.FullIsoTime(pendingDeletedAt)), "auto_delete_at") + } + return extra +} + +func (self *SDisk) GetExtraDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict { + extra := self.SSharableVirtualResourceBase.GetExtraDetails(ctx, userCred, query) + return self.getMoreDetails(extra) +} + +func (self *SDisk) GetCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict { + extra := self.SSharableVirtualResourceBase.GetCustomizeColumns(ctx, userCred, query) + return self.getMoreDetails(extra) +} + func (self *SDisk) StartDiskResizeTask(ctx context.Context, userCred mcclient.TokenCredential, size int64, parentTaskId string, pendingUsage quotas.IQuota) error { params := jsonutils.NewDict() params.Add(jsonutils.NewInt(size), "size") @@ -781,9 +882,8 @@ func (self *SDisk) GetAttachedGuests() []SGuest { q = q.Filter(sqlchemy.Equals(guestdisks.Field("disk_id"), self.Id)) ret := make([]SGuest, 0) - err := q.All(&ret) - if err != nil { - log.Errorf("%s", err) + if err := db.FetchModelObjects(GuestManager, q, &ret); err != nil { + log.Errorf("Fetch Geusts Objects %v", err) return nil } return ret @@ -820,6 +920,11 @@ func (self *SDisk) GetShortDesc() *jsonutils.JSONDict { storage := self.GetStorage() desc.Add(jsonutils.NewString(storage.StorageType), "storage_type") desc.Add(jsonutils.NewString(storage.MediumType), "medium_type") + + if priceKey := self.GetMetadata("price_key", nil); len(priceKey) > 0 { + desc.Add(jsonutils.NewString(priceKey), "price_key") + } + fs := self.GetFsFormat() if len(fs) > 0 { desc.Add(jsonutils.NewString(fs), "fs_format") diff --git a/pkg/compute/models/dnsrecords.go b/pkg/compute/models/dnsrecords.go index a1177a3401..5da570532c 100644 --- a/pkg/compute/models/dnsrecords.go +++ b/pkg/compute/models/dnsrecords.go @@ -6,7 +6,6 @@ import ( "strings" "yunion.io/x/jsonutils" - "yunion.io/x/log" "yunion.io/x/pkg/util/regutils" "yunion.io/x/sqlchemy" @@ -205,7 +204,6 @@ func (man *SDnsRecordManager) QueryDns(projectId, name string) *SDnsRecord { err := q.First(rec) if err != nil { - log.Errorf("QueryDns %q fail: %s", name, err) return nil } return rec diff --git a/pkg/compute/models/guestdrivers.go b/pkg/compute/models/guestdrivers.go index 9e9ce6c70c..48406df24f 100644 --- a/pkg/compute/models/guestdrivers.go +++ b/pkg/compute/models/guestdrivers.go @@ -39,6 +39,9 @@ type IGuestDriver interface { RequestGuestCreateInsertIso(ctx context.Context, imageId string, guest *SGuest, task taskman.ITask) error StartGuestStopTask(guest *SGuest, ctx context.Context, userCred mcclient.TokenCredential, params *jsonutils.JSONDict, parentTaskId string) error + StartGuestResetTask(guest *SGuest, ctx context.Context, userCred mcclient.TokenCredential, isHard bool, parentTaskId string) error + + RequestSoftReset(ctx context.Context, guest *SGuest, task taskman.ITask) error RequestDeployGuestOnHost(ctx context.Context, guest *SGuest, host *SHost, task taskman.ITask) error diff --git a/pkg/compute/models/guestnetworks.go b/pkg/compute/models/guestnetworks.go index 5ff6910060..882bbb5851 100644 --- a/pkg/compute/models/guestnetworks.go +++ b/pkg/compute/models/guestnetworks.go @@ -19,6 +19,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/compute/options" + "database/sql" ) const ( @@ -263,17 +264,18 @@ func (self *SGuestnetwork) GetJsonDescAtHost(host *SHost) jsonutils.JSONObject { } func (manager *SGuestnetworkManager) GetGuestByAddress(address string) *SGuest { - gn := SGuestnetwork{} - gn.SetModelManager(GuestnetworkManager) - - err := manager.Query().Equals("ip_addr", address).First(&gn) - if err != nil { - log.Errorf("GetGuestByAddress query fail: %s", err) - return nil - } - guest, _ := GuestManager.FetchById(gn.GuestId) - if guest != nil { - return guest.(*SGuest) + networks := manager.TableSpec().Instance() + guests := GuestManager.Query() + q := guests.Join(networks, sqlchemy.AND( + sqlchemy.IsFalse(networks.Field("deleted")), + sqlchemy.Equals(networks.Field("ip_addr"), address), + sqlchemy.Equals(networks.Field("guest_id"), guests.Field("id")), + )) + guest := &SGuest{} + guest.SetModelManager(GuestManager) + err := q.First(guest) + if err == nil { + return guest } return nil } @@ -333,6 +335,22 @@ func (manager *SGuestnetworkManager) DeleteGuestNics(ctx context.Context, guest return nil } +func (manager *SGuestnetworkManager) getGuestNicByIP(ip string) (*SGuestnetwork, error) { + gn := SGuestnetwork{} + q := manager.Query() + q = q.Equals("ip_addr", ip) + err := q.First(&gn) + if err != nil { + if err != sql.ErrNoRows { + log.Errorf("getGuestNicByIP fail %s", err) + return nil, err + } + return nil, nil + } + gn.SetModelManager(manager) + return &gn, nil +} + func (self *SGuestnetwork) LogDetachEvent(userCred mcclient.TokenCredential, guest *SGuest, network *SNetwork) { if network == nil { netTmp, _ := NetworkManager.FetchById(self.NetworkId) diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 49eef954de..7baa389fe2 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -205,49 +205,60 @@ func (manager *SGuestManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQ if len(hypervisor) > 0 { q = q.Equals("hypervisor", hypervisor) } + hostFilter, _ := queryDict.GetString("host") - zoneFilter, _ := queryDict.GetString("zone") - wireFilter, _ := queryDict.GetString("wire") - networkFilter, _ := queryDict.GetString("network") - diskFilter, _ := queryDict.GetString("disk") - var sq *sqlchemy.SSubQuery if len(hostFilter) > 0 { host, _ := HostManager.FetchByIdOrName("", hostFilter) if host == nil { - return nil, httperrors.NewResourceNotFoundError(fmt.Sprintf("host %s not found", hostFilter)) + return nil, httperrors.NewResourceNotFoundError("host %s not found", hostFilter) } - sq = HostManager.Query("id").Equals("id", host.GetId()).SubQuery() - } else if len(zoneFilter) > 0 { + q = q.Equals("host_id", host.GetId()) + } + + zoneFilter, _ := queryDict.GetString("zone") + if len(zoneFilter) > 0 { zone, _ := ZoneManager.FetchByIdOrName("", zoneFilter) if zone == nil { - return nil, httperrors.NewResourceNotFoundError(fmt.Sprintf("zone %s not found", zoneFilter)) + return nil, httperrors.NewResourceNotFoundError("zone %s not found", zoneFilter) } hostTable := HostManager.Query().SubQuery() zoneTable := ZoneManager.Query().SubQuery() - sq = hostTable.Query(hostTable.Field("id")).Join(zoneTable, + sq := hostTable.Query(hostTable.Field("id")).Join(zoneTable, sqlchemy.Equals(zoneTable.Field("id"), hostTable.Field("zone_id"))).Filter(sqlchemy.Equals(zoneTable.Field("id"), zone.GetId())).SubQuery() - } else if len(wireFilter) > 0 { + q = q.In("host_id", sq) + } + + wireFilter, _ := queryDict.GetString("wire") + if len(wireFilter) > 0 { wire, _ := WireManager.FetchByIdOrName("", wireFilter) if wire == nil { - return nil, httperrors.NewResourceNotFoundError(fmt.Sprintf("wire %s not found", wireFilter)) + return nil, httperrors.NewResourceNotFoundError("wire %s not found", wireFilter) } hostTable := HostManager.Query().SubQuery() hostWire := HostwireManager.Query().SubQuery() - sq = hostTable.Query(hostTable.Field("id")).Join(hostWire, sqlchemy.Equals(hostWire.Field("host_id"), hostTable.Field("id"))).Filter(sqlchemy.Equals(hostWire.Field("wire_id"), wire.GetId())).SubQuery() - } else if len(networkFilter) > 0 { + sq := hostTable.Query(hostTable.Field("id")).Join(hostWire, sqlchemy.Equals(hostWire.Field("host_id"), hostTable.Field("id"))).Filter(sqlchemy.Equals(hostWire.Field("wire_id"), wire.GetId())).SubQuery() + q = q.In("host_id", sq) + } + + networkFilter, _ := queryDict.GetString("network") + if len(networkFilter) > 0 { netI, _ := NetworkManager.FetchByIdOrName(userCred.GetProjectId(), networkFilter) if netI == nil { - return nil, httperrors.NewResourceNotFoundError(fmt.Sprintf("network %s not found", networkFilter)) + return nil, httperrors.NewResourceNotFoundError("network %s not found", networkFilter) } net := netI.(*SNetwork) hostTable := HostManager.Query().SubQuery() hostWire := HostwireManager.Query().SubQuery() - sq = hostTable.Query(hostTable.Field("id")).Join(hostWire, + sq := hostTable.Query(hostTable.Field("id")).Join(hostWire, sqlchemy.Equals(hostWire.Field("host_id"), hostTable.Field("id"))).Filter(sqlchemy.Equals(hostWire.Field("wire_id"), net.WireId)).SubQuery() - } else if len(diskFilter) > 0 { + q = q.In("host_id", sq) + } + + diskFilter, _ := queryDict.GetString("disk") + if len(diskFilter) > 0 { diskI, _ := DiskManager.FetchByIdOrName(userCred.GetProjectId(), diskFilter) if diskI == nil { - return nil, httperrors.NewResourceNotFoundError(fmt.Sprintf("disk %s not found", diskFilter)) + return nil, httperrors.NewResourceNotFoundError("disk %s not found", diskFilter) } disk := diskI.(*SDisk) guestdisks := GuestdiskManager.Query().SubQuery() @@ -264,7 +275,7 @@ func (manager *SGuestManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQ hosts := HostManager.Query().SubQuery() hoststorages := HoststorageManager.Query().SubQuery() storages := StorageManager.Query().SubQuery() - sq = hosts.Query(hosts.Field("id")). + sq := hosts.Query(hosts.Field("id")). Join(hoststorages, sqlchemy.AND( sqlchemy.Equals(hoststorages.Field("host_id"), hosts.Field("id")), sqlchemy.IsFalse(hoststorages.Field("deleted")))). @@ -272,11 +283,21 @@ func (manager *SGuestManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQ sqlchemy.Equals(storages.Field("id"), hoststorages.Field("storage_id")), sqlchemy.IsFalse(storages.Field("deleted")))). Filter(sqlchemy.Equals(storages.Field("id"), disk.StorageId)).SubQuery() + q = q.In("host_id", sq) } } - if sq != nil { + + managerFilter, _ := queryDict.GetString("manager") + if len(managerFilter) > 0 { + managerI, _ := CloudproviderManager.FetchByIdOrName(userCred.GetProjectId(), managerFilter) + if managerI == nil { + return nil, httperrors.NewResourceNotFoundError("cloud provider %s not found", managerFilter) + } + hosts := HostManager.Query().SubQuery() + sq := hosts.Query(hosts.Field("id")).Equals("manager_id", managerI.GetId()).SubQuery() q = q.In("host_id", sq) } + gpu, _ := queryDict.GetString("gpu") if len(gpu) != 0 { isodev := IsolatedDeviceManager.Query().SubQuery() @@ -472,9 +493,11 @@ func (self *SGuest) ValidateUpdateData(ctx context.Context, userCred mcclient.To return nil, err } - // if data.Contains("name") { - // return nil, httperrors.NewInputParameterError("cannot update server name") - // } + if data.Contains("name") { + if name, _ := data.GetString("name"); len(name) < 2 { + return nil, httperrors.NewInputParameterError("name is to short") + } + } /* if self.GetHypervisor() == HYPERVISOR_BAREMETAL { return nil, httperrors.NewInputParameterError("Cannot modify memory for baremetal") } @@ -906,11 +929,7 @@ func (self *SGuest) GetCustomizeColumns(ctx context.Context, userCred mcclient.T // extra.Add(jsonutils.NewString(self.getFlavorName()), "flavor") extra.Add(jsonutils.NewString(self.getKeypairName()), "keypair") extra.Add(jsonutils.NewInt(int64(self.getExtBandwidth())), "ext_bw") - zone := self.getZone() - if zone != nil { - extra.Add(jsonutils.NewString(zone.Id), "zone_id") - extra.Add(jsonutils.NewString(zone.Name), "zone") - } + extra.Add(jsonutils.NewString(self.GetSecgroupName()), "secgroup") if self.PendingDeleted { @@ -918,6 +937,28 @@ func (self *SGuest) GetCustomizeColumns(ctx context.Context, userCred mcclient.T extra.Add(jsonutils.NewString(timeutils.FullIsoTime(pendingDeletedAt)), "auto_delete_at") } + return self.moreExtraInfo(extra) +} + +func (self *SGuest) moreExtraInfo(extra *jsonutils.JSONDict) *jsonutils.JSONDict { + zone := self.getZone() + if zone != nil { + extra.Add(jsonutils.NewString(zone.GetId()), "zone_id") + extra.Add(jsonutils.NewString(zone.GetName()), "zone") + if len(zone.ExternalId) > 0 { + extra.Add(jsonutils.NewString(zone.ExternalId), "zone_external_id") + } + + region := zone.GetRegion() + if region != nil { + extra.Add(jsonutils.NewString(region.Id), "region_id") + extra.Add(jsonutils.NewString(region.Name), "region") + + if len(region.ExternalId) > 0 { + extra.Add(jsonutils.NewString(region.ExternalId), "region_external_id") + } + } + } return extra } @@ -948,14 +989,7 @@ func (self *SGuest) GetExtraDetails(ctx context.Context, userCred mcclient.Token } extra.Add(jsonutils.NewString(self.getAdminSecurityRules()), "admin_security_rules") } - zone := self.getZone() - if zone != nil { - extra.Add(jsonutils.NewString(zone.GetId()), "zone_id") - extra.Add(jsonutils.NewString(zone.GetName()), "zone") - extra.Add(jsonutils.NewString(zone.GetRegion().GetName()), "region") - extra.Add(jsonutils.NewString(zone.GetRegion().GetId()), "region_id") - } - return extra + return self.moreExtraInfo(extra) } func (self *SGuest) getNetworksDetails() string { @@ -1215,6 +1249,18 @@ func (self *SGuest) syncWithCloudVM(ctx context.Context, userCred mcclient.Token db.OpsLog.LogEvent(self, db.ACT_UPDATE, diffStr, userCred) } } + if metaData := extVM.GetMetadata(); metaData != nil { + meta := make(map[string]string, 0) + if metaData.Unmarshal(meta); err != nil { + log.Errorf("Get VM Metadata error: %v", err) + } else { + for key, value := range meta { + if err := self.SetMetadata(ctx, key, value, userCred); err != nil { + log.Errorf("set guest %s mata %s => %s error: %v", self.Name, key, value, err) + } + } + } + } return nil } @@ -1245,6 +1291,20 @@ func (manager *SGuestManager) newCloudVM(ctx context.Context, userCred mcclient. if err != nil { log.Errorf("Insert fail %s", err) } + + if metaData := extVM.GetMetadata(); metaData != nil { + meta := make(map[string]string, 0) + if err := metaData.Unmarshal(meta); err != nil { + log.Errorf("Get VM Metadata error: %v", err) + } else { + for key, value := range meta { + if err := guest.SetMetadata(ctx, key, value, userCred); err != nil { + log.Errorf("set guest %s mata %s => %s error: %v", guest.Name, key, value, err) + } + } + } + } + return &guest, nil } @@ -1443,7 +1503,20 @@ func (self *SGuest) SyncVMNics(ctx context.Context, userCred mcclient.TokenCrede if add.net == nil { continue // cannot determine which network it attached to } - err := self.Attach2Network(ctx, userCred, add.net, nil, add.nic.GetIP(), + // check if the IP has been occupied, if yes, release the IP + gn, err := GuestnetworkManager.getGuestNicByIP(add.nic.GetIP()) + if err != nil { + result.AddError(err) + continue + } + if gn != nil { + err = gn.Detach(ctx, userCred) + if err != nil { + result.AddError(err) + continue + } + } + err = self.Attach2Network(ctx, userCred, add.net, nil, add.nic.GetIP(), add.nic.GetMAC(), add.nic.GetDriver(), 0, false, -1, add.reserve, IPAllocationDefault, true) if err != nil { result.AddError(err) @@ -1507,7 +1580,11 @@ func (self *SGuest) PerformDeploy(ctx context.Context, userCred mcclient.TokenCr if !ok { return nil, fmt.Errorf("Parse query body error") } + + // 变更密码/密钥时需要Restart才能生效。更新普通字段不需要Restart + doRestart := false if kwargs.Contains("__delete_keypair__") || kwargs.Contains("keypair") { + doRestart = true var kpId string if !jsonutils.QueryBoolean(kwargs, "__delete_keypair__", false) { keypair, _ := kwargs.GetString("keypair") @@ -1529,8 +1606,9 @@ func (self *SGuest) PerformDeploy(ctx context.Context, userCred mcclient.TokenCr kwargs.Set("reset_password", jsonutils.JSONTrue) } } + if utils.IsInStringArray(self.Status, []string{VM_RUNNING, VM_READY, VM_ADMIN}) { - if self.Status == VM_RUNNING { + if doRestart && self.Status == VM_RUNNING { kwargs.Set("restart", jsonutils.JSONTrue) } err := self.StartGuestDeployTask(ctx, userCred, kwargs, "deploy", "") @@ -1627,7 +1705,7 @@ func (self *SGuest) SyncVMDisks(ctx context.Context, userCred mcclient.TokenCred if len(vdisks[i].GetGlobalId()) == 0 { continue } - disk, err := DiskManager.syncCloudDisk(userCred, vdisks[i]) + disk, err := DiskManager.syncCloudDisk(ctx, userCred, vdisks[i]) if err != nil { result.Error(err) return result @@ -1885,12 +1963,10 @@ func (self *SGuest) createDiskOnHost(ctx context.Context, userCred mcclient.Toke if storage == nil { return nil, fmt.Errorf("No storage to create disk") } - disk, err := self.createDiskOnStorage(ctx, userCred, storage, diskConfig, pendingUsage) if err != nil { return nil, err } - err = self.attach2Disk(disk, userCred, diskConfig.Driver, diskConfig.Cache, diskConfig.Mountpoint) return disk, err } @@ -2316,7 +2392,7 @@ func (self *SGuest) PerformDetachdisk(ctx context.Context, userCred mcclient.Tok disk.SetStatus(userCred, DISK_DETACHING, "") } taskData := jsonutils.NewDict() - taskData.Add(jsonutils.NewString(diskId), "disk_id") + taskData.Add(jsonutils.NewString(disk.Id), "disk_id") taskData.Add(jsonutils.NewBool(keepDisk), "keep_disk") self.GetDriver().StartGuestDetachdiskTask(ctx, userCred, self, taskData, "") return nil, nil @@ -2423,16 +2499,27 @@ func (self *SGuest) PerformChangeConfig(ctx context.Context, userCred mcclient.T diskIdx += 1 } - for storageId, needSize := range diskSizes { - iStorage, err := StorageManager.FetchById(storageId) - if err != nil { - return nil, httperrors.NewBadRequestError("Fetch storage error: %s", err) - } - storage := iStorage.(*SStorage) - if storage.GetFreeCapacity() < needSize { - return nil, httperrors.NewInsufficientResourceError("Not enough free space") - } + provider, e := self.GetHost().GetDriver() + if e != nil { + log.Errorf("Get Provider Error: %s", e) + return nil, httperrors.NewInsufficientResourceError("Provider Not Found") } + + if !provider.IsPublicCloud() { + for storageId, needSize := range diskSizes { + iStorage, err := StorageManager.FetchById(storageId) + if err != nil { + return nil, httperrors.NewBadRequestError("Fetch storage error: %s", err) + } + storage := iStorage.(*SStorage) + if storage.GetFreeCapacity() < needSize { + return nil, httperrors.NewInsufficientResourceError("Not enough free space") + } + } + } else { + log.Debugf("Skip storage free capacity validating for public cloud: %s", provider.GetName()) + } + if newDisks.Length() > 0 { confs.Add(newDisks, "create") } @@ -2592,9 +2679,9 @@ func (self *SGuest) AllowDeleteItem(ctx context.Context, userCred mcclient.Token func (self *SGuest) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error { overridePendingDelete := false purge := false - if data != nil { - overridePendingDelete = jsonutils.QueryBoolean(data, "override_pending_delete", false) - purge = jsonutils.QueryBoolean(data, "purge", false) + if query != nil { + overridePendingDelete = jsonutils.QueryBoolean(query, "override_pending_delete", false) + purge = jsonutils.QueryBoolean(query, "purge", false) } return self.StartDeleteGuestTask(ctx, userCred, "", purge, overridePendingDelete) } @@ -3006,6 +3093,111 @@ func (manager *SGuestManager) FetchGuestById(guestId string) *SGuest { return guest.(*SGuest) } +func (self *SGuest) GetSpec(checkStatus bool) *jsonutils.JSONDict { + if checkStatus { + if !utils.IsInStringArray(self.Status, []string{VM_SCHEDULE_FAILED}) { + return nil + } + } + spec := jsonutils.NewDict() + spec.Set("cpu", jsonutils.NewInt(int64(self.VcpuCount))) + spec.Set("mem", jsonutils.NewInt(int64(self.VmemSize))) + + // get disk spec + guestdisks := self.GetDisks() + diskSpecs := jsonutils.NewArray() + for _, guestdisk := range guestdisks { + disk := guestdisk.GetDisk() + diskSpec := jsonutils.NewDict() + diskSpec.Set("size", jsonutils.NewInt(int64(disk.DiskSize))) + s := disk.GetStorage() + diskSpec.Set("backend", jsonutils.NewString(s.StorageType)) + diskSpec.Set("medium_type", jsonutils.NewString(s.MediumType)) + diskSpecs.Add(diskSpec) + } + spec.Set("disk", diskSpecs) + + // get nic spec + guestnics := self.GetNetworks() + nicSpecs := jsonutils.NewArray() + for _, guestnic := range guestnics { + nicSpec := jsonutils.NewDict() + nicSpec.Set("bandwidth", jsonutils.NewInt(int64(guestnic.getBandwidth()))) + t := "int" + if guestnic.IsExit() { + t = "ext" + } + nicSpec.Set("type", jsonutils.NewString(t)) + nicSpecs.Add(nicSpec) + } + spec.Set("nic", nicSpecs) + + // get isolate device spec + guestgpus := self.GetIsolatedDevices() + gpuSpecs := jsonutils.NewArray() + for _, guestgpu := range guestgpus { + if strings.HasPrefix(guestgpu.DevType, "GPU") { + gs := guestgpu.GetSpec(false) + if gs != nil { + gpuSpecs.Add(gs) + } + } + } + spec.Set("gpu", gpuSpecs) + return spec +} + +func (self *SGuest) GetTemplateId() string { + guestdisks := self.GetDisks() + for _, guestdisk := range guestdisks { + disk := guestdisk.GetDisk() + if disk != nil { + templateId := disk.GetTemplateId() + if len(templateId) > 0 { + return templateId + } + } + } + return "" +} + +func (self *SGuest) GetShortDesc() *jsonutils.JSONDict { + desc := self.SStandaloneResourceBase.GetShortDesc() + desc.Set("mem", jsonutils.NewInt(int64(self.VmemSize))) + desc.Set("cpu", jsonutils.NewInt(int64(self.VcpuCount))) + templateId := self.GetTemplateId() + if len(templateId) > 0 { + desc.Set("cpu", jsonutils.NewString(templateId)) + } + extBw := self.getBandwidth(true) + intBw := self.getBandwidth(false) + if extBw > 0 { + desc.Set("ext_bandwidth", jsonutils.NewInt(int64(extBw))) + } + if intBw > 0 { + desc.Set("int_bandwidth", jsonutils.NewInt(int64(intBw))) + } + + if priceKey := self.GetMetadata("price_key", nil); len(priceKey) > 0 { + desc.Add(jsonutils.NewString(priceKey), "price_key") + } + + desc.Set("hypervisor", jsonutils.NewString(self.GetHypervisor())) + spec := self.GetSpec(false) + if self.GetHypervisor() == HYPERVISOR_BAREMETAL { + host := self.GetHost() + if host != nil { + hostSpec := host.GetSpec(false) + hostSpecIdent := host.GetSpecIdent(hostSpec) + spec.Set("host_spec", jsonutils.NewString(strings.Join(hostSpecIdent, "/"))) + } + } + if spec != nil { + desc.Update(spec) + } + return desc +} + func (self *SGuest) saveOsType(osType string) error { _, err := self.GetModelManager().TableSpec().Update(self, func() error { self.OsType = osType @@ -3114,6 +3306,23 @@ func (self *SGuest) PerformStart(ctx context.Context, userCred mcclient.TokenCre } } +func (self *SGuest) AllowPerformReset(ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + data jsonutils.JSONObject) bool { + return self.IsOwner(userCred) +} + +func (self *SGuest) PerformReset(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, + data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + isHard := jsonutils.QueryBoolean(data, "is_hard", false) + if self.Status == VM_RUNNING || self.Status == VM_STOP_FAILED { + self.GetDriver().StartGuestResetTask(self, ctx, userCred, isHard, "") + return nil, nil + } + return nil, httperrors.NewInvalidStatusError("Cannot reset VM in status %s", self.Status) +} + func (self *SGuest) AllowPerformStop(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, diff --git a/pkg/compute/models/hostdrivers.go b/pkg/compute/models/hostdrivers.go index 3f162d75da..6749e5ac68 100644 --- a/pkg/compute/models/hostdrivers.go +++ b/pkg/compute/models/hostdrivers.go @@ -12,7 +12,7 @@ import ( type IHostDriver interface { GetHostType() string CheckAndSetCacheImage(ctx context.Context, host *SHost, storagecache *SStoragecache, scimg *SStoragecachedimage, task taskman.ITask) error - RequestAllocateDiskOnStorage(host *SHost, storage *SStorage, disk *SDisk, task taskman.ITask, content *jsonutils.JSONDict) error + RequestAllocateDiskOnStorage(ctx context.Context, host *SHost, storage *SStorage, disk *SDisk, task taskman.ITask, content *jsonutils.JSONDict) error RequestDeallocateDiskOnHost(host *SHost, storage *SStorage, disk *SDisk, task taskman.ITask) error RequestResizeDiskOnHostOnline(host *SHost, storage *SStorage, disk *SDisk, size int64, task taskman.ITask) error RequestResizeDiskOnHost(host *SHost, storage *SStorage, disk *SDisk, size int64, task taskman.ITask) error diff --git a/pkg/compute/models/hostnetworks.go b/pkg/compute/models/hostnetworks.go index abf31b3ed6..5a94918e32 100644 --- a/pkg/compute/models/hostnetworks.go +++ b/pkg/compute/models/hostnetworks.go @@ -92,7 +92,7 @@ func (man *SHostnetworkManager) QueryByAddress(addr string) *sqlchemy.SQuery { func (man *SHostnetworkManager) GetHostNetworkByAddress(addr string) *SHostnetwork { network := SHostnetwork{} err := man.QueryByAddress(addr).First(&network) - if err != nil { + if err == nil { return &network } return nil @@ -107,9 +107,18 @@ func (man *SHostnetworkManager) GetNetworkByAddress(addr string) *SNetwork { } func (man *SHostnetworkManager) GetHostByAddress(addr string) *SHost { - net := man.GetHostNetworkByAddress(addr) - if net == nil { - return nil + networks := man.TableSpec().Instance() + hosts := HostManager.Query() + q := hosts.Join(networks, sqlchemy.AND( + sqlchemy.IsFalse(networks.Field("deleted")), + sqlchemy.Equals(networks.Field("ip_addr"), addr), + sqlchemy.Equals(networks.Field("baremetal_id"), hosts.Field("id")), + )) + host := &SHost{} + host.SetModelManager(HostManager) + err := q.First(host) + if err == nil { + return host } - return net.GetHost() + return nil } diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index 6b47c4e370..0874133ba1 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -506,6 +506,58 @@ func (self *SHost) ClearSchedDescCache() error { return HostManager.ClearSchedDescCache(self.Id) } +func (self *SHost) GetSpec(statusCheck bool) *jsonutils.JSONDict { + if statusCheck { + if utils.IsInStringArray(self.Status, []string{BAREMETAL_INIT, BAREMETAL_PREPARE_FAIL, BAREMETAL_PREPARE}) || + self.getBaremetalServer() != nil { + return nil + } + if self.MemSize == 0 || self.CpuCount == 0 { + return nil + } + } + spec := self.GetHardwareSpecification() + spec.Remove("storage_info") + netInfo := jsonutils.NewArray() + nifs := self.GetNetInterfaces() + for _, nif := range nifs { + netDesc := nif.getBaremetalJsonDesc() + nicType, err := netDesc.GetString("nic_type") + if err != nil && nicType != NIC_TYPE_IPMI { + netInfo.Add(netDesc) + } + } + spec.Set("nic_count", jsonutils.NewInt(int64(netInfo.Length()))) + manufacture, err := self.SysInfo.Get("manufacture") + if err != nil { + manufacture = jsonutils.NewString("Unknown") + } + spec.Set("manufacture", manufacture) + model, err := self.SysInfo.Get("model") + if err != nil { + model = jsonutils.NewString("Unknown") + } + spec.Set("model", model) + return nil +} + +func (self *SHost) GetSpecIdent(spec *jsonutils.JSONDict) []string { + // Todo + return []string{} +} + +func (self *SHost) GetHardwareSpecification() *jsonutils.JSONDict { + spec := jsonutils.NewDict() + spec.Set("cpu", jsonutils.NewInt(int64(self.CpuCount))) + spec.Set("mem", jsonutils.NewInt(int64(self.MemSize))) + if self.StorageInfo != nil { + spec.Set("disk", GetDiskSpecV2(self.StorageInfo)) + spec.Set("driver", jsonutils.NewString(self.StorageDriver)) + spec.Set("storage_info", self.StorageInfo) + } + return spec +} + type SStorageCapacity struct { Capacity int Used int diff --git a/pkg/compute/models/isolated_devices.go b/pkg/compute/models/isolated_devices.go index f3182d3f67..d1c7c26c21 100644 --- a/pkg/compute/models/isolated_devices.go +++ b/pkg/compute/models/isolated_devices.go @@ -380,6 +380,24 @@ func (self *SIsolatedDevice) getDesc() *jsonutils.JSONDict { return desc } +func (self *SIsolatedDevice) GetSpec(checkStatus bool) *jsonutils.JSONDict { + if checkStatus { + if len(self.GuestId) > 0 { + return nil + } + host := self.getHost() + if host.Status != BAREMETAL_RUNNING || !host.Enabled { + return nil + } + } + spec := jsonutils.NewDict() + spec.Set("dev_type", jsonutils.NewString(self.DevType)) + spec.Set("model", jsonutils.NewString(self.Model)) + spec.Set("pci_id", jsonutils.NewString(self.VendorDeviceId)) + spec.Set("vendor", jsonutils.NewString(self.getVendor())) + return spec +} + func (self *SIsolatedDevice) GetShortDesc() *jsonutils.JSONDict { desc := self.getDesc() desc.Add(jsonutils.NewString(self.Keyword()), "res_name") diff --git a/pkg/compute/models/secgrouprules.go b/pkg/compute/models/secgrouprules.go index 4347ea48c9..46b032e195 100644 --- a/pkg/compute/models/secgrouprules.go +++ b/pkg/compute/models/secgrouprules.go @@ -328,7 +328,7 @@ func (manager *SSecurityGroupRuleManager) SyncRules(ctx context.Context, userCre cmp := strings.Compare(dbStr, ruleStr) if cmp == 0 { if dbRules[j].Description != rules[i].Description { - if _, err := manager.TableSpec().Update(dbRules[j], func() error { + if _, err := manager.TableSpec().Update(&dbRules[j], func() error { dbRules[j].Description = rules[i].Description return nil }); err != nil { diff --git a/pkg/compute/models/storages.go b/pkg/compute/models/storages.go index 4ac8ea7587..0093f4de8b 100644 --- a/pkg/compute/models/storages.go +++ b/pkg/compute/models/storages.go @@ -12,6 +12,7 @@ import ( "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/pkg/tristate" "yunion.io/x/pkg/util/compare" + "yunion.io/x/pkg/util/sysutils" "yunion.io/x/sqlchemy" ) @@ -100,6 +101,10 @@ func (self *SStorage) IsLocal() bool { return self.StorageType == STORAGE_LOCAL || self.StorageType == STORAGE_BAREMETAL } +func (manager *SStorageManager) AllowListItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool { + return true +} + func (self *SStorage) GetUsedCapacity(isReady tristate.TriState) int { disks := DiskManager.Query().SubQuery() q := disks.Query(sqlchemy.SUM("sum", disks.Field("disk_size"))).Equals("storage_id", self.Id) @@ -654,6 +659,26 @@ func (manager *SStorageManager) ListItemFilter(ctx context.Context, q *sqlchemy. q = q.Filter(sqlchemy.In(q.Field("zone_id"), sq.SubQuery())) } + if jsonutils.QueryBoolean(query, "share", false) { + q = q.Filter(sqlchemy.NotIn(q.Field("storage_type"), sysutils.LOCAL_STORAGE_TYPES)) + } + + if jsonutils.QueryBoolean(query, "local", false) { + q = q.Filter(sqlchemy.In(q.Field("storage_type"), sysutils.LOCAL_STORAGE_TYPES)) + } + + if jsonutils.QueryBoolean(query, "usable", false) { + hostStorageTable := HoststorageManager.Query().SubQuery() + hostTable := HostManager.Query().SubQuery() + sq := hostStorageTable.Query(hostStorageTable.Field("storage_id")).Join(hostTable, + sqlchemy.Equals(hostTable.Field("id"), hostStorageTable.Field("host_id"))). + Filter(sqlchemy.Equals(hostTable.Field("host_status"), HOST_ONLINE)) + + q = q.Filter(sqlchemy.In(q.Field("id"), sq)). + Filter(sqlchemy.In(q.Field("status"), []string{STORAGE_ENABLED, STORAGE_ONLINE})). + Filter(sqlchemy.IsTrue(q.Field("enabled"))) + } + managerStr := jsonutils.GetAnyString(query, []string{"manager", "provider", "manager_id", "provider_id"}) if len(managerStr) > 0 { provider := CloudproviderManager.FetchCloudproviderByIdOrName(managerStr) diff --git a/pkg/compute/models/wires.go b/pkg/compute/models/wires.go index 22fa6d9a6f..d522c53e58 100644 --- a/pkg/compute/models/wires.go +++ b/pkg/compute/models/wires.go @@ -347,6 +347,7 @@ func (self *SWire) getNetworks() ([]SNetwork, error) { func (self *SWire) getGatewayNetworkQuery() *sqlchemy.SQuery { q := self.getNetworkQuery() q = q.IsNotNull("guest_gateway").IsNotEmpty("guest_gateway") + q = q.Equals("status", NETWORK_STATUS_AVAILABLE) return q } diff --git a/pkg/compute/models/zones.go b/pkg/compute/models/zones.go index 05fc01a0c9..fe4379b0de 100644 --- a/pkg/compute/models/zones.go +++ b/pkg/compute/models/zones.go @@ -515,7 +515,7 @@ func (self *SZone) GetDetailsCapabilities(ctx context.Context, userCred mcclient } func (self *SZone) getHypervisors() []string { - q := HostManager.Query("host_type").Equals("zone_id", self.Id) + q := HostManager.Query("host_type").Equals("zone_id", self.Id).Distinct() rows, err := q.Rows() if err != nil { return nil diff --git a/pkg/compute/options/options.go b/pkg/compute/options/options.go index c873819ebd..a018a3e2f1 100644 --- a/pkg/compute/options/options.go +++ b/pkg/compute/options/options.go @@ -21,10 +21,10 @@ type ComputeOptions struct { DefaultDiskSize int `default:"30720" help:"Default disk size in MB if not specified, default to 30GiB"` - EnablePendingDelete bool `default:"true" help:"Turn on/off pending delete VM and disk, default is on"` - PendingDeleteCheckSeconds int `default:"3600" help:"How long to wait to scan pending delete VM or disks, default is 1 hour"` - PendingDeleteExpireSeconds int `default:"259200" help:"How long a pending delete VM/disks cleaned automatically, default 3 days"` - PendingDeleteMaxCleanBatchSize int `default:"50" help:"How many pending delete servers can be clean in a batch"` + EnablePendingDelete bool `default:"true" help:"Turn on/off pending delete VM and disk, default is on"` + PendingDeleteCheckSeconds int `default:"3600" help:"How long to wait to scan pending delete VM or disks, default is 1 hour"` + PendingDeleteExpireSeconds int `default:"259200" help:"How long a pending delete VM/disks cleaned automatically, default 3 days"` + PendingDeleteMaxCleanBatchSize int `default:"50" help:"How many pending delete servers can be clean in a batch"` ImageCacheStoragePolicy string `default:"least_used" choices:"best_fit|least_used" help:"Policy to choose storage for image cache, best_fit or least_used"` MetricsRetentionDays int32 `default:"30" help:"Retention days for monitoring metrics in influxdb"` diff --git a/pkg/compute/tasks/disk_create_task.go b/pkg/compute/tasks/disk_create_task.go index 1c47d60d5a..be0e2635ed 100644 --- a/pkg/compute/tasks/disk_create_task.go +++ b/pkg/compute/tasks/disk_create_task.go @@ -40,10 +40,10 @@ func (self *DiskCreateTask) OnStorageCacheImageComplete(ctx context.Context, dis } storage := disk.GetStorage() host := storage.GetMasterHost() - db.OpsLog.LogEvent(disk, db.ACT_ALLOCATE, disk.GetShortDesc(), self.GetUserCred()) + db.OpsLog.LogEvent(disk, db.ACT_ALLOCATING, disk.GetShortDesc(), self.GetUserCred()) disk.SetStatus(self.GetUserCred(), models.DISK_STARTALLOC, "") self.SetStage("on_disk_ready", nil) - if err := disk.StartAllocate(host, storage, self.GetTaskId(), self.GetUserCred(), rebuild, snapshot, self); err != nil { + if err := disk.StartAllocate(ctx, host, storage, self.GetTaskId(), self.GetUserCred(), rebuild, snapshot, self); err != nil { self.OnStartAllocateFailed(ctx, disk, jsonutils.NewString(err.Error())) } } diff --git a/pkg/compute/tasks/disk_delete_task.go b/pkg/compute/tasks/disk_delete_task.go index 436165f427..ff6c4f76c1 100644 --- a/pkg/compute/tasks/disk_delete_task.go +++ b/pkg/compute/tasks/disk_delete_task.go @@ -8,6 +8,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/compute/options" + "yunion.io/x/onecloud/pkg/httperrors" ) type DiskDeleteTask struct { @@ -51,7 +52,9 @@ func (self *DiskDeleteTask) startDeleteDisk(ctx context.Context, disk *models.SD self.OnGuestDiskDeleteSucc(ctx, disk, nil) } else { self.SetStage("on_guest_disk_delete_succ", nil) - if err := host.GetHostDriver().RequestDeallocateDiskOnHost(host, storage, disk, self); err != nil { + if host == nil { + self.OnGuestDiskDeleteFailed(ctx, disk, httperrors.NewNotFoundError("fail to find master host")) + } else if err := host.GetHostDriver().RequestDeallocateDiskOnHost(host, storage, disk, self); err != nil { self.OnGuestDiskDeleteFailed(ctx, disk, err) } } diff --git a/pkg/compute/tasks/disk_resize_task.go b/pkg/compute/tasks/disk_resize_task.go index 584c43ab94..dd25da5879 100644 --- a/pkg/compute/tasks/disk_resize_task.go +++ b/pkg/compute/tasks/disk_resize_task.go @@ -32,7 +32,7 @@ func (self *DiskResizeTask) OnInit(ctx context.Context, obj db.IStandaloneModel, } resion := "Cannot find host for disk" if host == nil || host.HostStatus != models.HOST_ONLINE { - disk.SetStatus(self.GetUserCred(), models.DISK_READY, resion) + disk.SetDiskReady(ctx, self.GetUserCred(), resion) self.SetStageFailed(ctx, resion) db.OpsLog.LogEvent(disk, db.ACT_RESIZE_FAIL, resion, self.GetUserCred()) } else { @@ -65,7 +65,7 @@ func (self *DiskResizeTask) OnStartResizeDiskSucc(ctx context.Context, disk *mod } func (self *DiskResizeTask) OnStartResizeDiskFailed(ctx context.Context, disk *models.SDisk, resion error) { - disk.SetStatus(self.GetUserCred(), models.DISK_READY, resion.Error()) + disk.SetDiskReady(ctx, self.GetUserCred(), resion.Error()) self.SetStageFailed(ctx, resion.Error()) db.OpsLog.LogEvent(disk, db.ACT_RESIZE_FAIL, resion.Error(), self.GetUserCred()) } @@ -94,6 +94,7 @@ func (self *DiskResizeTask) OnDiskResizeComplete(ctx context.Context, disk *mode self.OnStartResizeDiskFailed(ctx, disk, err) return } + disk.SetDiskReady(ctx, self.GetUserCred(), "") notes := fmt.Sprintf("%s=>%s", oldStatus, disk.Status) db.OpsLog.LogEvent(disk, db.ACT_UPDATE_STATUS, notes, self.UserCred) self.CleanHostSchedCache(disk) @@ -103,6 +104,6 @@ func (self *DiskResizeTask) OnDiskResizeComplete(ctx context.Context, disk *mode } func (self *DiskResizeTask) OnDiskResizeCompleteFailed(ctx context.Context, disk *models.SDisk, resion error) { - disk.SetStatus(self.UserCred, models.DISK_READY, resion.Error()) + disk.SetDiskReady(ctx, self.GetUserCred(), resion.Error()) db.OpsLog.LogEvent(disk, db.ACT_RESIZE_FAIL, disk.GetShortDesc(), self.UserCred) } diff --git a/pkg/compute/tasks/guest_create_disk_task.go b/pkg/compute/tasks/guest_create_disk_task.go index c7034bd735..bc734944cd 100644 --- a/pkg/compute/tasks/guest_create_disk_task.go +++ b/pkg/compute/tasks/guest_create_disk_task.go @@ -9,6 +9,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/log" ) type GuestCreateDiskTask struct { @@ -118,7 +119,115 @@ func (self *KVMGuestCreateDiskTask) OnConfigSyncComplete(ctx context.Context, ob self.SetStageComplete(ctx, nil) } +type AliyunGuestCreateDiskTask struct { + SGuestBaseTask +} + +func (self *AliyunGuestCreateDiskTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + self.SetStage("on_aliyun_disk_prepared", nil) + self.OnAliyunDiskPrepared(ctx, obj, data) +} + +func (self *AliyunGuestCreateDiskTask) OnAliyunDiskPrepared(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + var diskIndex = 0 + var diskReady = true + for { + diskId, err := self.Params.GetString(fmt.Sprintf("disk.%d.id", diskIndex)) + if !diskReady || err != nil { + break + } + iDisk, err := models.DiskManager.FetchById(diskId) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + if iDisk == nil { + self.SetStageFailed(ctx, "Disk not found") + return + } + disk := iDisk.(*models.SDisk) + if disk.Status == models.DISK_INIT { + snapInfo, err := self.Params.GetString(fmt.Sprintf("disk.%d.snapshot", diskIndex)) + if err != nil { + snapInfo = "" + } + err = disk.StartDiskCreateTask(ctx, self.UserCred, false, snapInfo, self.GetTaskId()) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + diskReady = false + break + } + diskIndex += 1 + } + diskIndex = 0 + guest := obj.(*models.SGuest) + + for { + diskId, err := self.Params.GetString(fmt.Sprintf("disk.%d.id", diskIndex)) + if !diskReady || err != nil { + break + } + iDisk, err := models.DiskManager.FetchById(diskId) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + if iDisk == nil { + self.SetStageFailed(ctx, "Disk not found") + return + } + disk := iDisk.(*models.SDisk) + if disk.Status != models.DISK_READY { + diskReady = false + break + } + + ihost, err := guest.GetHost().GetIHost() + if err != nil { + self.SetStageFailed(ctx, "Host not found") + return + } + + iVM, e := ihost.GetIVMById(guest.GetExternalId()) + if e != nil { + self.SetStageFailed(ctx, "Aliyun VM not found") + return + } + + err = iVM.AttachDisk(disk.GetExternalId()) + if err != nil { + log.Debugf("Attach Disk %s to guest fail: %s", diskId, err) + self.SetStageFailed(ctx, "Attach Disk to guest fail") + return + } + diskIndex += 1 + } + + if diskReady { + if guest.Status == models.VM_RUNNING { + self.SetStage("on_config_sync_complete", nil) + err := guest.StartSyncstatus(ctx, self.UserCred, self.GetTaskId()) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + } + } else { + self.SetStageComplete(ctx, nil) + } + } +} + +func (self *AliyunGuestCreateDiskTask) OnConfigSyncComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + self.SetStageComplete(ctx, nil) +} + +func (self *AliyunGuestCreateDiskTask) AttachAliyunDisks(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + self.SetStageComplete(ctx, nil) +} + func init() { taskman.RegisterTask(GuestCreateDiskTask{}) taskman.RegisterTask(KVMGuestCreateDiskTask{}) + taskman.RegisterTask(AliyunGuestCreateDiskTask{}) } diff --git a/pkg/compute/tasks/guest_delete_task.go b/pkg/compute/tasks/guest_delete_task.go index b86a63d2ec..bd9ce9eec1 100644 --- a/pkg/compute/tasks/guest_delete_task.go +++ b/pkg/compute/tasks/guest_delete_task.go @@ -4,12 +4,14 @@ import ( "context" "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/utils" + "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/compute/options" - "yunion.io/x/pkg/utils" ) type GuestDeleteTask struct { @@ -28,15 +30,18 @@ func (self *GuestDeleteTask) OnInit(ctx context.Context, obj db.IStandaloneModel func (self *GuestDeleteTask) OnGuestStopComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { guest := obj.(*models.SGuest) - guestStatus, _ := self.Params.GetString("guest_status") if options.Options.EnablePendingDelete && !guest.PendingDeleted && !jsonutils.QueryBoolean(self.Params, "purge", false) && - !jsonutils.QueryBoolean(self.Params, "override_pending_delete", false) && - !utils.IsInStringArray(guestStatus, []string{models.VM_SCHEDULE_FAILED, models.VM_NETWORK_FAILED, models.VM_DISK_FAILED, + !jsonutils.QueryBoolean(self.Params, "override_pending_delete", false) { + log.Debugf("XXXXXXX Do guest pending delete... XXXXXXX") + guestStatus, _ := self.Params.GetString("guest_status") + if !utils.IsInStringArray(guestStatus, []string{models.VM_SCHEDULE_FAILED, models.VM_NETWORK_FAILED, models.VM_DISK_FAILED, models.VM_CREATE_FAILED, models.VM_DEVICE_FAILED}) { - self.StartPendingDeleteGuest(ctx, guest) - return + self.StartPendingDeleteGuest(ctx, guest) + return + } } + log.Debugf("XXXXXXX Do real delete on guest ... XXXXXXX") self.OnGuestStopCompleteFailed(ctx, guest, data) } diff --git a/pkg/compute/tasks/guest_deploy_task.go b/pkg/compute/tasks/guest_deploy_task.go index 47fd39b7c7..037a909a1c 100644 --- a/pkg/compute/tasks/guest_deploy_task.go +++ b/pkg/compute/tasks/guest_deploy_task.go @@ -54,14 +54,10 @@ func (self *GuestDeployTask) StartDeployGuestOnHost(ctx context.Context, guest * log.Errorf("request_deploy_guest_on_host %s", err) self.OnDeployGuestFail(ctx, guest, err) } else { - self.OnDeployGuestSucc(guest) + guest.SetStatus(self.UserCred, models.VM_DEPLOYING, "") } } -func (self *GuestDeployTask) OnDeployGuestSucc(guest *models.SGuest) { - guest.SetStatus(self.UserCred, models.VM_DEPLOYING, "") -} - func (self *GuestDeployTask) OnDeployGuestFail(ctx context.Context, guest *models.SGuest, err error) { guest.SetStatus(self.UserCred, models.VM_DEPLOY_FAILED, err.Error()) self.SetStageFailed(ctx, err.Error()) diff --git a/pkg/compute/tasks/guest_rebuild_root_task.go b/pkg/compute/tasks/guest_rebuild_root_task.go index 0d0ee7ee6a..50de80c0e9 100644 --- a/pkg/compute/tasks/guest_rebuild_root_task.go +++ b/pkg/compute/tasks/guest_rebuild_root_task.go @@ -32,6 +32,10 @@ func (self *GuestRebuildRootTask) OnInit(ctx context.Context, obj db.IStandalone } } +func (self *GuestRebuildRootTask) OnStopServerComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + self.StartRebuildRootDisk(ctx, guest) +} + func (self *GuestRebuildRootTask) StartRebuildRootDisk(ctx context.Context, guest *models.SGuest) { db.OpsLog.LogEvent(guest, db.ACT_REBUILDING_ROOT, nil, self.UserCred) gds := guest.CategorizeDisks() @@ -137,4 +141,4 @@ func (self *KVMGuestRebuildRootTask) OnRebuildRootDiskCompleteFailed(ctx context func (self *KVMGuestRebuildRootTask) OnGuestDeployComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { self.SetStageComplete(ctx, nil) -} +} \ No newline at end of file diff --git a/pkg/compute/tasks/guest_reset_task.go b/pkg/compute/tasks/guest_reset_task.go new file mode 100644 index 0000000000..72a5273e2d --- /dev/null +++ b/pkg/compute/tasks/guest_reset_task.go @@ -0,0 +1,58 @@ +package tasks + +import ( + "context" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" +) + +func init() { + taskman.RegisterTask(GuestSoftResetTask{}) + taskman.RegisterTask(GuestHardResetTask{}) +} + +type GuestSoftResetTask struct { + SGuestBaseTask +} + +func (self *GuestSoftResetTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + err := guest.GetDriver().RequestSoftReset(ctx, guest, self) + if err == nil { + self.SetStageComplete(ctx, nil) + } else { + self.SetStageFailed(ctx, err.Error()) + } +} + +type GuestHardResetTask struct { + SGuestBaseTask +} + +func (self *GuestHardResetTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + self.StopServer(ctx, guest) +} + +func (self *GuestHardResetTask) StopServer(ctx context.Context, guest *models.SGuest) { + guest.SetStatus(self.UserCred, models.VM_STOPPING, "") + self.SetStage("OnServerStopComplete", nil) + guest.StartGuestStopTask(ctx, self.UserCred, false, self.GetTaskId()) +} + +func (self *GuestHardResetTask) OnServerStopComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + self.StartServer(ctx, guest) +} + +func (self *GuestHardResetTask) StartServer(ctx context.Context, guest *models.SGuest) { + self.SetStage("OnServerStartComplete", nil) + guest.StartGueststartTask(ctx, self.UserCred, nil, self.GetTaskId()) +} + +func (self *GuestHardResetTask) OnServerStartComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + self.SetStageComplete(ctx, nil) +} diff --git a/pkg/compute/tasks/guest_start_task.go b/pkg/compute/tasks/guest_start_task.go index 7bbb2ca2dc..4e2cad8135 100644 --- a/pkg/compute/tasks/guest_start_task.go +++ b/pkg/compute/tasks/guest_start_task.go @@ -73,6 +73,7 @@ func (self *GuestStartTask) OnStartCompleteFailed(ctx context.Context, obj db.IS } func (self *GuestStartTask) onStartGuestFailed(ctx context.Context, guest *models.SGuest, err error) { + guest.SetStatus(self.UserCred, models.VM_START_FAILED, err.Error()) self.SetStageFailed(ctx, err.Error()) self.OnStartCompleteFailed(ctx, guest, jsonutils.NewString(err.Error())) } diff --git a/pkg/dns/README.md b/pkg/dns/README.md new file mode 100644 index 0000000000..79bfeb66f3 --- /dev/null +++ b/pkg/dns/README.md @@ -0,0 +1,56 @@ +# 行为约定 + +A + +```sh +names='' +names="$names titan" #ok, guest PlainName +names="$names titan.hq.cloud.yunionyun.com" #ok, guest CloudZoneFQDN +names="$names kubenode" #ok, host PlainName +names="$names kubenode.hq.cloud.yunionyun.com" #ok, host CloudZoneFQDN +names="$names whoever-the-ether" #NXDOMAIN, nonexistent PlainName +names="$names whoever-the-ether.titan.hq.cloud.yunionyun.com" #NXDOMAIN, nonexistent CloudZoneFQDN +names="$names mail.google.com" #ok, dnsrecords +names="$names www.douban.com" #ok, pub +names="$names app" #ok, k8s svc PlainName in "default" ns +names="$names app.default" #NXDOMAIN, k8s svc name.namespace +names="$names app.default.hq.cloud.yunionyun.com" #ok, k8s name.ns CloudZoneFQDN +names="$names mon-kafka.system" #NXDOMAIN, k8s svc name.namespace +names="$names mon-kafka.system.hq.cloud.yunionyun.com" #ok, k8s name.ns CloudZoneFQDN + +# TODO source IP +# TODO other TYPEs +for name in $names; do + echo "############### $name" + #dig @192.168.222.171 $name + #dig -p 54 @10.168.222.136 $name + dig -p 54 @192.168.222.171 $name +done +``` + +PTR + +```sh +names='' +names="$names " #ok, guest ip +names="$names " #ok, host ip +names="$names " #ok, ptr records in db +names="$names " #NXDOMAIN, others + +for name in $names; do + echo "############### $name" + dig -p 54 @192.168.222.171 -x $name +done +``` + +# 配置 + + log { + # note that apart from rcode like NXDOMAIN, SERVFAIL, coredns will also + # log NOERROR response when it's NoData as defined by coredns itself + # + # > NoData indicates name found, but not the type: NOERROR in header, SOA in auth. + # + class denial + class error + } diff --git a/pkg/dns/dns.go b/pkg/dns/dns.go index b60ab73c51..248504b9f6 100644 --- a/pkg/dns/dns.go +++ b/pkg/dns/dns.go @@ -18,25 +18,27 @@ import ( "github.com/mholt/caddy" "github.com/miekg/dns" v1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes" ylog "yunion.io/x/log" - "yunion.io/x/pkg/utils" - "yunion.io/x/sqlchemy" - "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/k8s" + "yunion.io/x/pkg/utils" + "yunion.io/x/sqlchemy" ) const ( PluginName string = "yunion" // defaultTTL to apply to all answers - defaultTTL = 10 + defaultTTL = 10 + defaultDbMaxOpenConn = 32 + defaultDbMaxIdleConn = 32 ) var ( @@ -74,15 +76,17 @@ func (r *SRegionDNS) initDB(c *caddy.Controller) error { if err != nil { return err } - dbConn, err := sql.Open(dialect, sqlStr) + sqlDb, err := sql.Open(dialect, sqlStr) if err != nil { return err } - sqlchemy.SetDB(dbConn) + sqlDb.SetMaxOpenConns(defaultDbMaxOpenConn) + sqlDb.SetMaxIdleConns(defaultDbMaxIdleConn) + sqlchemy.SetDB(sqlDb) db.InitAllManagers() c.OnShutdown(func() error { - r.CloseDB() + sqlchemy.CloseDB() return nil }) return nil @@ -103,10 +107,6 @@ func (r *SRegionDNS) initK8s(c *caddy.Controller) { ylog.Infof("Init k8s client success, %d pods in the cluster", len(pods.Items)) } -func (r *SRegionDNS) CloseDB() { - sqlchemy.CloseDB() -} - func (r *SRegionDNS) ServeDNS(ctx context.Context, w dns.ResponseWriter, rmsg *dns.Msg) (int, error) { var ( records []dns.RR @@ -116,36 +116,26 @@ func (r *SRegionDNS) ServeDNS(ctx context.Context, w dns.ResponseWriter, rmsg *d opt := plugin.Options{} state := request.Request{W: w, Req: rmsg, Context: ctx} - zone := plugin.Zones(r.Zones).Matches(state.Name()) - switch state.QType() { case dns.TypeA: - ylog.Debugf("A question: %#v", state) records, err = plugin.A(r, zone, state, nil, opt) case dns.TypeAAAA: - ylog.Debugf("AAAA question: %#v", state) + // TODO fallthrough to next records, err = plugin.AAAA(r, zone, state, nil, opt) case dns.TypeTXT: - ylog.Debugf("TXT question: %#v", state) records, err = plugin.TXT(r, zone, state, opt) case dns.TypeCNAME: - ylog.Debugf("CNAME question: %#v", state) records, err = plugin.CNAME(r, zone, state, opt) case dns.TypePTR: - ylog.Debugf("PTR question: %#v", state) records, err = plugin.PTR(r, zone, state, opt) case dns.TypeMX: - ylog.Debugf("MX question: %#v", state) records, extra, err = plugin.MX(r, zone, state, opt) case dns.TypeSRV: - ylog.Debugf("SRV question: %#v", state) records, extra, err = plugin.SRV(r, zone, state, opt) case dns.TypeSOA: - ylog.Debugf("SOA question: %#v", state) records, err = plugin.SOA(r, zone, state, opt) case dns.TypeNS: - ylog.Debugf("NS question: %#v", state) if state.Name() == zone { records, extra, err = plugin.NS(r, zone, state, opt) break @@ -157,18 +147,19 @@ func (r *SRegionDNS) ServeDNS(ctx context.Context, w dns.ResponseWriter, rmsg *d _, err = plugin.A(r, zone, state, nil, opt) } - if r.IsNameError(err) { + if err == errCallNext { if r.Fall.Through(state.Name()) { return plugin.NextOrFailure(r.Name(), r.Next, ctx, w, rmsg) } return plugin.BackendError(r, zone, dns.RcodeNameError, state, nil /* err */, opt) - } - if err != nil { - return plugin.BackendError(r, zone, dns.RcodeServerFailure, state, err, opt) + } else if err == errRefused { + return plugin.BackendError(r, zone, dns.RcodeRefused, state, err, opt) + } else if err == errNotFound { + return plugin.BackendError(r, zone, dns.RcodeNameError, state, err, opt) } if len(records) == 0 { - return plugin.BackendError(r, zone, dns.RcodeSuccess, state, err, opt) + return plugin.BackendError(r, zone, dns.RcodeNameError, state, err, opt) } m := new(dns.Msg) @@ -184,13 +175,14 @@ func (r *SRegionDNS) ServeDNS(ctx context.Context, w dns.ResponseWriter, rmsg *d } var ( - errNoItems = errors.New("no items found") + errRefused = errors.New("refused the query") + errNotFound = errors.New("not found") + errCallNext = errors.New("continue to next") ) // Services implements the ServiceBackend interface func (r *SRegionDNS) Services(state request.Request, exact bool, opt plugin.Options) (services []msg.Service, err error) { switch state.QType() { - case dns.TypeTXT: t, _ := dnsutil.TrimZone(state.Name(), state.Zone) @@ -203,7 +195,6 @@ func (r *SRegionDNS) Services(state request.Request, exact bool, opt plugin.Opti } svc := msg.Service{Text: "0.0.1", TTL: 28800, Key: msg.Path(state.QName(), "coredns")} return []msg.Service{svc}, nil - case dns.TypeNS: ns := r.nsAddr() svc := msg.Service{Host: ns.A.String(), Key: msg.Path(state.QName(), "coredns")} @@ -219,7 +210,6 @@ func (r *SRegionDNS) Services(state request.Request, exact bool, opt plugin.Opti } services, err = r.Records(state, false) - ylog.Debugf("Get records: %#v, error: %v", services, err) return } @@ -230,7 +220,7 @@ func (r *SRegionDNS) Lookup(state request.Request, name string, typ uint16) (*dn // IsNameError implements the ServiceBackend interface func (r *SRegionDNS) IsNameError(err error) bool { - return err == errNoItems + return err == errCallNext } // Records looks up records in region mysql @@ -242,25 +232,22 @@ func (r *SRegionDNS) Records(state request.Request, exact bool) ([]msg.Service, return r.findRecords(req) } -func (r *SRegionDNS) getHostIpWithName(req *recordRequest) []string { +func (r *SRegionDNS) getHostIpWithName(req *recordRequest) string { name := req.QueryName() host, _ := models.HostManager.FetchByName("", name) if host == nil { - return nil + return "" } ip := host.(*models.SHost).AccessIp - if len(ip) == 0 { - return nil - } - return []string{ip} + return ip } func (r *SRegionDNS) getGuestIpWithName(req *recordRequest) []string { ips := []string{} name := req.QueryName() projectId := req.ProjectId() - isExitOnly := req.IsExitOnly() - ips = models.GuestManager.GetIpInProjectWithName(projectId, name, isExitOnly) + wantOnlyExit := false + ips = models.GuestManager.GetIpInProjectWithName(projectId, name, wantOnlyExit) return ips } @@ -268,6 +255,9 @@ func (r *SRegionDNS) getK8sServiceBackends(req *recordRequest) ([]string, error) queryInfo := req.GetK8sQueryInfo() pods, err := r.getK8sServicePods(queryInfo.Namespace, queryInfo.ServiceName) if err != nil { + if k8serrors.IsNotFound(err) { + err = nil + } return nil, err } ips := make([]string, 0) @@ -281,7 +271,11 @@ func (r *SRegionDNS) getK8sServiceBackends(req *recordRequest) ([]string, error) } func (r *SRegionDNS) getK8sServicePods(namespace, name string) ([]v1.Pod, error) { - cli := r.K8sClient + cli, err := k8s.NewClientByFile(r.K8sConfigFile, nil) + if err != nil { + ylog.Errorf("Init kubernetes client error: %v", err) + return nil, err + } svc, err := cli.CoreV1().Services(namespace).Get(name, metav1.GetOptions{}) if err != nil { return nil, err @@ -301,10 +295,9 @@ func (r *SRegionDNS) Name() string { return PluginName } -func (r *SRegionDNS) queryLocalDnsRecords(req *recordRequest) (recs []msg.Service, err error) { +func (r *SRegionDNS) queryLocalDnsRecords(req *recordRequest) (recs []msg.Service) { ips := models.DnsRecordManager.QueryDnsIps(req.ProjectId(), req.Name(), req.Type()) if len(ips) == 0 { - err = errNoItems return } @@ -317,12 +310,12 @@ func (r *SRegionDNS) queryLocalDnsRecords(req *recordRequest) (recs []msg.Servic if req.IsSRV() { parts := strings.SplitN(ip.Addr, ":", 2) if len(parts) != 2 { - err = fmt.Errorf("Invalid SRV records: %q", ip.Addr) + ylog.Errorf("Invalid SRV records: %q", ip.Addr) return } port, e := strconv.Atoi(parts[1]) if e != nil { - err = e + ylog.Errorf("Invalid SRV records: %q", ip.Addr) return } s = msg.Service{Host: parts[0], Port: port, TTL: ttl} @@ -334,17 +327,6 @@ func (r *SRegionDNS) queryLocalDnsRecords(req *recordRequest) (recs []msg.Servic return } -func (r *SRegionDNS) IsCloudNetworkIp(req *recordRequest) bool { - if req.network != nil { - return true - } - return false -} - -func (r *SRegionDNS) IsK8sClientReady() bool { - return r.K8sClient != nil -} - func (r *SRegionDNS) isMyDomain(req *recordRequest) bool { zones := []string{fmt.Sprintf("%s.", r.PrimaryZone)} zone := plugin.Zones(zones).Matches(req.state.Name()) @@ -354,54 +336,65 @@ func (r *SRegionDNS) isMyDomain(req *recordRequest) bool { return false } -func (r *SRegionDNS) findRecords(req *recordRequest) (recs []msg.Service, err error) { +func (r *SRegionDNS) findRecords(req *recordRequest) ([]msg.Service, error) { // 1. try local dns records table - recs, err = r.queryLocalDnsRecords(req) - if len(recs) != 0 { - return + rrs := r.queryLocalDnsRecords(req) + if len(rrs) > 0 { + return rrs, nil } + isPlainName := req.IsPlainName() isMyDomain := r.isMyDomain(req) - - isCloudIp := r.IsCloudNetworkIp(req) - - // 2. not my domain and src ip not in cloud network table - // query from upstream - if !isMyDomain && !isCloudIp { - err = errNoItems - return + if isPlainName { + isCloudIp := req.SrcInCloud() + if isCloudIp { + ips := r.findInternalRecordIps(req) + if len(ips) > 0 { + return ips2DnsRecords(ips), nil + } else { + return nil, errNotFound + } + } else { + return nil, errRefused + } + } else if isMyDomain { + ips := r.findInternalRecordIps(req) + if len(ips) > 0 { + return ips2DnsRecords(ips), nil + } else { + return nil, errNotFound + } + } else { + return nil, errCallNext } - - // 3. internal query - ips, err := r.findInternalRecordIps(req) - return ips2DnsRecords(ips), err } -func (r *SRegionDNS) findInternalRecordIps(req *recordRequest) ([]string, error) { - // 1. try host table - ip := r.getHostIpWithName(req) - if len(ip) != 0 { - return ip, nil +func (r *SRegionDNS) findInternalRecordIps(req *recordRequest) []string { + { + // 1. try host table + ip := r.getHostIpWithName(req) + if len(ip) > 0 { + return []string{ip} + } } - // 2. try guest table - ip = r.getGuestIpWithName(req) - if len(ip) != 0 { - return ip, nil + { + // 2. try guest table + ips := r.getGuestIpWithName(req) + if len(ips) > 0 { + return ips + } } - if !r.IsK8sClientReady() { + if r.K8sClient == nil { ylog.Warningf("K8s client not ready, skip it.") - return nil, errNoItems + return nil } // 3. try k8s service backends ips, err := r.getK8sServiceBackends(req) - if len(ips) != 0 { - return ips, nil - } if err != nil { ylog.Errorf("Get k8s service backends error: %v", err) } - return nil, errNoItems + return ips } func ips2DnsRecords(ips []string) []msg.Service { diff --git a/pkg/dns/parse.go b/pkg/dns/parse.go index 5cf836b646..568a773567 100644 --- a/pkg/dns/parse.go +++ b/pkg/dns/parse.go @@ -7,17 +7,16 @@ import ( "github.com/coredns/coredns/request" "github.com/miekg/dns" - "yunion.io/x/pkg/tristate" - "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/pkg/tristate" ) type recordRequest struct { - state request.Request - domainSegs []string - guest *models.SGuest - host *models.SHost - network *models.SNetwork + state request.Request + domainSegs []string + srcProjectId string + srcInCloud bool + network *models.SNetwork } func parseRequest(state request.Request) (r *recordRequest, err error) { @@ -28,9 +27,19 @@ func parseRequest(state request.Request) (r *recordRequest, err error) { domainSegs: segs, } srcIP := r.SrcIP4() - r.guest = models.GuestnetworkManager.GetGuestByAddress(srcIP) - r.host = models.HostnetworkManager.GetHostByAddress(srcIP) - r.network, _ = models.NetworkManager.GetNetworkOfIP(srcIP, "", tristate.None) + // NOTE the check on networks_tbl is a hack, we should be more specific + // by querying only guest networks, host networks, and others like + // loadbalancer network the to come. + // + // Order matters here, we want to find the srcIP project as accurately + // as possible + if guest := models.GuestnetworkManager.GetGuestByAddress(srcIP); guest != nil { + r.srcProjectId = guest.ProjectId + r.srcInCloud = true + } else if network, _ := models.NetworkManager.GetNetworkOfIP(srcIP, "", tristate.None); network != nil { + r.srcProjectId = network.ProjectId + r.srcInCloud = true + } return } @@ -41,6 +50,11 @@ func (r recordRequest) Name() string { return name } +func (r recordRequest) IsPlainName() bool { + nl := dns.CountLabel(r.Name()) + return nl == 1 +} + func (r recordRequest) QueryName() string { seps := strings.Split(r.Name(), ".") if len(seps) == 0 { @@ -63,20 +77,11 @@ func (r recordRequest) SrcIP4() string { } func (r recordRequest) ProjectId() string { - if r.guest != nil { - return r.guest.ProjectId - } - if r.network != nil { - return r.network.ProjectId - } - return "" + return r.srcProjectId } -func (r recordRequest) IsExitOnly() bool { - if r.guest == nil { - return false - } - return r.guest.IsExitOnly() +func (r recordRequest) SrcInCloud() bool { + return r.srcInCloud } type K8sQueryInfo struct { diff --git a/pkg/dns/reverse.go b/pkg/dns/reverse.go index e6cf7e2276..f6df515fdc 100644 --- a/pkg/dns/reverse.go +++ b/pkg/dns/reverse.go @@ -8,8 +8,6 @@ import ( "github.com/coredns/coredns/plugin/pkg/dnsutil" "github.com/coredns/coredns/request" - "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/compute/models" ) @@ -21,13 +19,7 @@ func (r *SRegionDNS) Reverse(state request.Request, exact bool, opt plugin.Optio return nil, e } records, err := r.getNameForIp(ip, state) - if err != nil { - log.Errorf("Reverse get name for ip: %v", err) - } - if len(records) == 0 { - return records, errNoItems - } - return records, nil + return records, err } func (r *SRegionDNS) getNameForIp(ip string, state request.Request) ([]msg.Service, error) { @@ -53,7 +45,7 @@ func (r *SRegionDNS) getNameForIp(ip string, state request.Request) ([]msg.Servi if guest != nil { return []msg.Service{{Host: r.joinDomain(guest.Name), TTL: defaultTTL}}, nil } - return nil, errNoItems + return nil, errNotFound } func (r *SRegionDNS) joinDomain(name string) string { diff --git a/pkg/dns/setup.go b/pkg/dns/setup.go index 1a6a1ff673..77a9cb1c4b 100644 --- a/pkg/dns/setup.go +++ b/pkg/dns/setup.go @@ -2,7 +2,6 @@ package dns import ( "fmt" - "os" "github.com/coredns/coredns/core/dnsserver" "github.com/coredns/coredns/plugin" @@ -20,18 +19,16 @@ func init() { } func setup(c *caddy.Controller) error { - os.Stderr = os.Stdout - rDNS, err := regionDNSParse(c) if err != nil { return plugin.Error(PluginName, err) } - if rDNS.PrimaryZone == "" { - return fmt.Errorf("dns_domain must provided") + if len(rDNS.PrimaryZone) == 0 { + return fmt.Errorf("dns_domain missing") } if !regutils.MatchDomainName(rDNS.PrimaryZone) { - return fmt.Errorf("dns_domain %q not match domain format", rDNS.PrimaryZone) + return fmt.Errorf("dns_domain %q invalid", rDNS.PrimaryZone) } err = rDNS.initDB(c) diff --git a/pkg/mcclient/auth/auth.go b/pkg/mcclient/auth/auth.go index 214be4b0ad..4e0957e66a 100644 --- a/pkg/mcclient/auth/auth.go +++ b/pkg/mcclient/auth/auth.go @@ -103,7 +103,7 @@ func (c *TokenCacheVerify) Verify(cli *mcclient.Client, adminToken, token string if err != nil { return nil, fmt.Errorf("Add %s credential to cache: %#v", cred.GetTokenString(), err) } - log.Infof("Add token: %s", cred) + // log.Debugf("Add token: %s", cred) return cred, nil } diff --git a/pkg/mcclient/catalog.go b/pkg/mcclient/catalog.go new file mode 100644 index 0000000000..1238bf6723 --- /dev/null +++ b/pkg/mcclient/catalog.go @@ -0,0 +1,6 @@ +package mcclient + +type IServiceCatalog interface { + GetServiceURL(service, region, zone, endpointType string) (string, error) + GetServiceURLs(service, region, zone, endpointType string) ([]string, error) +} diff --git a/pkg/mcclient/mcclient.go b/pkg/mcclient/mcclient.go index 4df0561160..b537ae1a40 100644 --- a/pkg/mcclient/mcclient.go +++ b/pkg/mcclient/mcclient.go @@ -10,36 +10,16 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/onecloud/pkg/util/httputils" + "yunion.io/x/log" ) -/*const ( - USER_AGENT = "yunioncloud-go/201708" -) - -var ( - red = color.New(color.FgRed, color.Bold).PrintlnFunc() - green = color.New(color.FgGreen, color.Bold).PrintlnFunc() - yellow = color.New(color.FgYellow, color.Bold).PrintlnFunc() - cyan = color.New(color.FgHiCyan, color.Bold).PrintlnFunc() -)*/ - -/*type JSONClientError struct { - Code int - Class string - Details string -} - -func (e *JSONClientError) Error() string { - return fmt.Sprintf("JSONClientError: %s %d %s", e.Details, e.Code, e.Class) -}*/ - type Client struct { authUrl string timeout int debug bool httpconn *http.Client - // serviceCatalog *KeystoneServiceCatalog + serviceCatalog IServiceCatalog } func NewClient(authUrl string, timeout int, debug bool, insecure bool) *Client { @@ -222,7 +202,7 @@ func (this *Client) _authV3(domainName, uname, passwd, projectId, projectName, t if len(tokenId) == 0 { return nil, fmt.Errorf("No X-Subject-Token in header") } - ret, err := UnmarshalV3Token(rbody, tokenId) + ret, err := this.unmarshalV3Token(rbody, tokenId) return ret, err } @@ -245,7 +225,7 @@ func (this *Client) _authV2(uname, passwd, tenantId, tenantName, token string) ( if err != nil { return nil, err } - return UnmarshalV2Token(rbody) + return this.unmarshalV2Token(rbody) } func (this *Client) Authenticate(uname, passwd, domainName, tenantName string) (TokenCredential, error) { @@ -255,16 +235,21 @@ func (this *Client) Authenticate(uname, passwd, domainName, tenantName string) ( return this._authV2(uname, passwd, "", tenantName, "") } -func UnmarshalV3Token(rbody jsonutils.JSONObject, tokenId string) (cred TokenCredential, err error) { +func (this *Client) unmarshalV3Token(rbody jsonutils.JSONObject, tokenId string) (cred TokenCredential, err error) { cred = &TokenCredentialV3{Id: tokenId} err = rbody.Unmarshal(cred) if err != nil { err = fmt.Errorf("Invalid response when unmarshal V3 Token: %v", err) } + cata := cred.GetServiceCatalog() + if cata == nil { + log.Fatalf("No srvice catalog avaiable") + } + this.serviceCatalog = cata return } -func UnmarshalV2Token(rbody jsonutils.JSONObject) (cred TokenCredential, err error) { +func (this *Client) unmarshalV2Token(rbody jsonutils.JSONObject) (cred TokenCredential, err error) { access, err := rbody.Get("access") if err == nil { cred = &TokenCredentialV2{} @@ -272,6 +257,11 @@ func UnmarshalV2Token(rbody jsonutils.JSONObject) (cred TokenCredential, err err if err != nil { err = fmt.Errorf("Invalid response when unmarshal V2 Token: %s", err) } + cata := cred.GetServiceCatalog() + if cata == nil { + log.Fatalf("No srvice catalog avaiable") + } + this.serviceCatalog = cata return } err = fmt.Errorf("Invalid response: no access object") @@ -286,7 +276,7 @@ func (this *Client) verifyV3(adminToken, token string) (TokenCredential, error) if err != nil { return nil, err } - return UnmarshalV3Token(rbody, token) + return this.unmarshalV3Token(rbody, token) } func (this *Client) verifyV2(adminToken, token string) (TokenCredential, error) { @@ -297,7 +287,7 @@ func (this *Client) verifyV2(adminToken, token string) (TokenCredential, error) if err != nil { return nil, err } - return UnmarshalV2Token(rbody) + return this.unmarshalV2Token(rbody) } func (this *Client) Verify(adminToken, token string) (cred TokenCredential, err error) { @@ -320,6 +310,13 @@ func (this *Client) SetProject(tenantId, tenantName string, token TokenCredentia } func (this *Client) NewSession(region, zone, endpointType string, token TokenCredential, apiVersion string) *ClientSession { + cata := token.GetServiceCatalog() + if this.serviceCatalog == nil { + if cata == nil { + log.Fatalf("Missing service catalog in token") + } + this.serviceCatalog = cata + } return &ClientSession{client: this, region: region, zone: zone, endpointType: endpointType, token: token, apiVersion: apiVersion, diff --git a/pkg/mcclient/modules/k8s/base.go b/pkg/mcclient/modules/k8s/base.go index 6c5eef1847..3b5fc20b61 100644 --- a/pkg/mcclient/modules/k8s/base.go +++ b/pkg/mcclient/modules/k8s/base.go @@ -1,13 +1,56 @@ package k8s import ( + "yunion.io/x/pkg/util/sets" + "yunion.io/x/onecloud/pkg/mcclient/modules" ) -func NewManager(keyword, keywordPlural string, columns, adminColumns []string) *modules.ResourceManager { +func NewManager(keyword, keywordPlural string, columns, adminColumns *Columns) *modules.ResourceManager { return &modules.ResourceManager{ - BaseManager: *modules.NewBaseManager("k8s", "", "", columns, adminColumns), + BaseManager: *modules.NewBaseManager("k8s", "", "", columns.Array(), adminColumns.Array()), Keyword: keyword, KeywordPlural: keywordPlural, } } + +type Columns struct { + cols []string +} + +func NewColumns(cols ...string) *Columns { + c := &Columns{cols: make([]string, 0)} + return c.Add(cols...) +} + +func (c *Columns) Add(cols ...string) *Columns { + for _, col := range cols { + c.add(col) + } + return c +} + +func (c *Columns) add(col string) *Columns { + src := sets.NewString(c.Array()...) + if src.Has(col) { + return c + } + c.cols = append(c.cols, col) + return c +} + +func (c Columns) Array() []string { + return c.cols +} + +func NewNamespaceCols(col ...string) *Columns { + return NewColumns("name", "namespace").Add(col...) +} + +func NewClusterCols(col ...string) *Columns { + return NewColumns("cluster").Add(col...) +} + +func NewResourceCols(col ...string) *Columns { + return NewColumns("name", "id").Add(col...) +} diff --git a/pkg/mcclient/modules/k8s/cluster.go b/pkg/mcclient/modules/k8s/cluster.go index df2df5ca39..46b7bb4fea 100644 --- a/pkg/mcclient/modules/k8s/cluster.go +++ b/pkg/mcclient/modules/k8s/cluster.go @@ -9,8 +9,8 @@ var ( ) func init() { - Clusters = NewManager("cluster", "clusters", - []string{"id", "name", "mode", "k8s_version", "status", "api_endpoint"}, - []string{}) + Clusters = NewManager("kube_cluster", "kube_clusters", + NewResourceCols("mode", "k8s_version", "status", "api_endpoint"), + NewColumns()) modules.Register(Clusters) } diff --git a/pkg/mcclient/modules/k8s/configmap.go b/pkg/mcclient/modules/k8s/configmap.go new file mode 100644 index 0000000000..4bfcbbafdf --- /dev/null +++ b/pkg/mcclient/modules/k8s/configmap.go @@ -0,0 +1,20 @@ +package k8s + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +var ConfigMaps *ConfigMapManager + +type ConfigMapManager struct { + modules.ResourceManager +} + +func init() { + ConfigMaps = &ConfigMapManager{ + ResourceManager: *NewManager( + "configmap", "configmaps", + NewNamespaceCols(), + NewClusterCols())} + modules.Register(ConfigMaps) +} diff --git a/pkg/mcclient/modules/k8s/deployment.go b/pkg/mcclient/modules/k8s/deployment.go new file mode 100644 index 0000000000..f56a2947ef --- /dev/null +++ b/pkg/mcclient/modules/k8s/deployment.go @@ -0,0 +1,20 @@ +package k8s + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +var Deployments *DeploymentManager + +type DeploymentManager struct { + modules.ResourceManager +} + +func init() { + Deployments = &DeploymentManager{ + ResourceManager: *NewManager( + "deployment", "deployments", + NewNamespaceCols("labels"), + NewClusterCols())} + modules.Register(Deployments) +} diff --git a/pkg/mcclient/modules/k8s/event.go b/pkg/mcclient/modules/k8s/event.go index f71c5541ef..584ddbc376 100644 --- a/pkg/mcclient/modules/k8s/event.go +++ b/pkg/mcclient/modules/k8s/event.go @@ -10,7 +10,7 @@ var ( func init() { Logs = NewManager("kube_event", "kube_events", - []string{"id", "ops_time", "obj_id", "obj_type", "obj_name", "user", "user_id", "tenant", "tenant_id", "owner_tenant_id", "action", "notes"}, - []string{}) + NewColumns("id", "ops_time", "obj_id", "obj_type", "obj_name", "user", "user_id", "tenant", "tenant_id", "owner_tenant_id", "action", "notes"), + NewColumns()) modules.Register(Logs) } diff --git a/pkg/mcclient/modules/k8s/helm.go b/pkg/mcclient/modules/k8s/helm.go new file mode 100644 index 0000000000..b3e04f21ea --- /dev/null +++ b/pkg/mcclient/modules/k8s/helm.go @@ -0,0 +1,17 @@ +package k8s + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +var ( + Repos *modules.ResourceManager +) + +func init() { + Repos = NewManager("repo", "repos", + NewResourceCols("url", "is_public", "source"), + NewColumns(), + ) + modules.Register(Repos) +} diff --git a/pkg/mcclient/modules/k8s/helm_chart.go b/pkg/mcclient/modules/k8s/helm_chart.go new file mode 100644 index 0000000000..522628793f --- /dev/null +++ b/pkg/mcclient/modules/k8s/helm_chart.go @@ -0,0 +1,16 @@ +package k8s + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +var ( + Charts *modules.ResourceManager +) + +func init() { + Charts = NewManager("chart", "charts", + NewResourceCols(), + NewColumns()) + modules.Register(Charts) +} diff --git a/pkg/mcclient/modules/k8s/helm_release.go b/pkg/mcclient/modules/k8s/helm_release.go new file mode 100644 index 0000000000..ef254c0a45 --- /dev/null +++ b/pkg/mcclient/modules/k8s/helm_release.go @@ -0,0 +1,23 @@ +package k8s + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +var ( + Releases *ReleaseManager +) + +type ReleaseManager struct { + modules.ResourceManager +} + +func init() { + Releases = &ReleaseManager{ + ResourceManager: *NewManager( + "release", "releases", + NewNamespaceCols(""), + NewColumns(), + )} + modules.Register(Releases) +} diff --git a/pkg/mcclient/modules/k8s/helm_tiller.go b/pkg/mcclient/modules/k8s/helm_tiller.go new file mode 100644 index 0000000000..ddb442561f --- /dev/null +++ b/pkg/mcclient/modules/k8s/helm_tiller.go @@ -0,0 +1,23 @@ +package k8s + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +var ( + Tiller *TillerManager +) + +type TillerManager struct { + modules.ResourceManager +} + +func init() { + Tiller = &TillerManager{ + ResourceManager: *NewManager( + "tiller", "tiller", + NewColumns(), + NewColumns(), + )} + modules.Register(Tiller) +} diff --git a/pkg/mcclient/modules/k8s/node.go b/pkg/mcclient/modules/k8s/node.go index 5b8cb0798b..630c0ffb8c 100644 --- a/pkg/mcclient/modules/k8s/node.go +++ b/pkg/mcclient/modules/k8s/node.go @@ -9,6 +9,6 @@ var ( ) func init() { - Nodes = NewManager("node", "nodes", []string{"id", "name", "cluster", "roles", "address", "status"}, []string{}) + Nodes = NewManager("kube_node", "kube_nodes", NewResourceCols("cluster", "roles", "address", "status"), NewColumns()) modules.Register(Nodes) } diff --git a/pkg/mcclient/modules/k8s/pod.go b/pkg/mcclient/modules/k8s/pod.go index 46417df838..218ef6a243 100644 --- a/pkg/mcclient/modules/k8s/pod.go +++ b/pkg/mcclient/modules/k8s/pod.go @@ -14,7 +14,7 @@ func init() { Pods = &PodManager{ ResourceManager: *NewManager( "pod", "pods", - []string{"id", "name", "cluster", "roles", "address", "status"}, - []string{})} + NewNamespaceCols("podIP", "status", "restartCount", "labels"), + NewClusterCols("nodeName"))} modules.Register(Pods) } diff --git a/pkg/mcclient/modules/k8s/raw.go b/pkg/mcclient/modules/k8s/raw.go new file mode 100644 index 0000000000..8853a18742 --- /dev/null +++ b/pkg/mcclient/modules/k8s/raw.go @@ -0,0 +1,95 @@ +package k8s + +import ( + "fmt" + "net/url" + "strings" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +var RawResource *RawResourceManager + +func init() { + RawResource = &RawResourceManager{ + serviceType: "k8s", + } +} + +type rawResourceContext struct { + kind string + name string + query jsonutils.JSONObject + ctxs []modules.ManagerContext +} + +func newRawResourceContext(kind, namespace, name string, query jsonutils.JSONObject, ctxs []modules.ManagerContext) *rawResourceContext { + nsQuery := getNamespaceQuery(namespace) + if query != nil { + nsQuery.Update(query) + } + ctx := &rawResourceContext{kind: kind, name: name, query: nsQuery, ctxs: ctxs} + return ctx +} + +func (ctx rawResourceContext) contextPath() string { + segs := make([]string, 0) + ctxs := ctx.ctxs + if ctxs != nil && len(ctxs) > 0 { + for _, c := range ctxs { + segs = append(segs, c.InstanceManager.KeyString()) + if len(c.InstanceId) > 0 { + segs = append(segs, url.PathEscape(c.InstanceId)) + } + } + } + segs = append(segs, "_raw", ctx.kind, ctx.name) + path := fmt.Sprintf("/%s", strings.Join(segs, "/")) + if ctx.query != nil { + qs := ctx.query.QueryString() + if len(qs) > 0 { + path = fmt.Sprintf("%s?%s", path, qs) + } + } + return path +} + +type RawResourceManager struct { + serviceType string +} + +func (m *RawResourceManager) request(s *mcclient.ClientSession, method string, path string, body jsonutils.JSONObject) (jsonutils.JSONObject, error) { + _, ret, err := s.JSONRequest(m.serviceType, "", method, path, nil, body) + return ret, err +} + +func getNamespaceQuery(namespace string) *jsonutils.JSONDict { + query := jsonutils.NewDict() + if namespace != "" { + query.Set("namespace", jsonutils.NewString(namespace)) + } + return query +} + +func (m *RawResourceManager) Get(s *mcclient.ClientSession, kind string, namespace string, name string, query jsonutils.JSONObject, ctxs []modules.ManagerContext) (jsonutils.JSONObject, error) { + ctx := newRawResourceContext(kind, namespace, name, query, ctxs) + return m.request(s, "GET", ctx.contextPath(), nil) +} + +func (m *RawResourceManager) Put(s *mcclient.ClientSession, kind string, namespace string, name string, body jsonutils.JSONObject, ctxs []modules.ManagerContext) error { + rawBytes := body.String() + newBody := jsonutils.NewDict() + newBody.Add(jsonutils.NewString(rawBytes), "raw") + ctx := newRawResourceContext(kind, namespace, name, nil, ctxs) + _, err := m.request(s, "PUT", ctx.contextPath(), newBody) + return err +} + +func (m *RawResourceManager) Delete(s *mcclient.ClientSession, kind string, namespace string, name string, query jsonutils.JSONObject, ctxs []modules.ManagerContext) error { + ctx := newRawResourceContext(kind, namespace, name, query, ctxs) + _, err := m.request(s, "DELETE", ctx.contextPath(), nil) + return err +} diff --git a/pkg/mcclient/modules/k8s/service.go b/pkg/mcclient/modules/k8s/service.go new file mode 100644 index 0000000000..7a153c4bc4 --- /dev/null +++ b/pkg/mcclient/modules/k8s/service.go @@ -0,0 +1,20 @@ +package k8s + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +var Services *ServiceManager + +type ServiceManager struct { + modules.ResourceManager +} + +func init() { + Services = &ServiceManager{ + ResourceManager: *NewManager( + "k8s_service", "k8s_services", + NewNamespaceCols("clusterIP", "selector", "internalEndpoint", "externalEndpoints"), + NewClusterCols())} + modules.Register(Services) +} diff --git a/pkg/mcclient/modules/mod_domains.go b/pkg/mcclient/modules/mod_domains.go index 84d57ffb34..e7be85cd5e 100644 --- a/pkg/mcclient/modules/mod_domains.go +++ b/pkg/mcclient/modules/mod_domains.go @@ -240,12 +240,7 @@ func (this *DomainManager) DoDomainConfigDelete(s *mcclient.ClientSession, param return ret, httperrors.NewResourceNotFoundError("找不到该认证域") } - driver, err := detail.GetString("driver") - if err != nil { - log.Errorf("got driver from domain detail error: %v", err) - return ret, httperrors.NewInternalServerError("服务器错误,获取认证协议失败,不允许删除") - } - + driver, _ := detail.GetString("driver") if driver != "ldap" { if result, err := UsersV3.List(s, params); err != nil { log.Errorf("user list got error: %v", err) diff --git a/pkg/mcclient/modules/modules.go b/pkg/mcclient/modules/modules.go index 2381f50232..1ba4171f98 100644 --- a/pkg/mcclient/modules/modules.go +++ b/pkg/mcclient/modules/modules.go @@ -2,10 +2,10 @@ package modules import ( "fmt" - "log" "strings" "yunion.io/x/jsonutils" + "yunion.io/x/log" "yunion.io/x/onecloud/pkg/mcclient" ) @@ -142,6 +142,17 @@ func _getJointKey(mod1 Manager, mod2 Manager) string { return fmt.Sprintf("%s-%s", mod1.KeyString(), mod2.KeyString()) } +func ensureModuleNotRegistered(mod, newMod BaseManagerInterface) { + modSvcType := mod.ServiceType() + newModSvcType := newMod.ServiceType() + if mod == newMod { + log.Fatalf("Module %#v duplicate registered, service type: %q", mod, modSvcType) + } + if modSvcType != newModSvcType { + log.Fatalf("Module %#v already registered, service type is %q.\nSo new module %#v can't be registered, service type is %q", mod, modSvcType, newMod, newModSvcType) + } +} + func _register(version string, mod BaseManagerInterface) { if modules == nil { modules = make(map[string]map[string][]BaseManagerInterface) @@ -156,9 +167,7 @@ func _register(version string, mod BaseManagerInterface) { mods = make([]BaseManagerInterface, 0) } for _, m := range mods { - if m == mod { - log.Fatal("Module already registered", mod) - } + ensureModuleNotRegistered(m, mod) } modtable[mod.KeyString()] = append(mods, mod) } @@ -179,7 +188,7 @@ func _registerJointModule(version string, mod BaseManagerInterface) { } for _, m := range jointMods { if m == jointMod { - log.Fatal("Module already registered", jointMod) + ensureModuleNotRegistered(m, jointMod) } } modtable[jointKey] = append(jointMods, jointMod) diff --git a/pkg/mcclient/session.go b/pkg/mcclient/session.go index 3526f6eea4..527e05c4e4 100644 --- a/pkg/mcclient/session.go +++ b/pkg/mcclient/session.go @@ -75,7 +75,26 @@ func (this *ClientSession) GetServiceURL(service, endpointType string) (string, if len(this.apiVersion) > 0 && this.apiVersion != DEFAULT_API_VERSION { service = fmt.Sprintf("%s_%s", service, this.apiVersion) } - return this.token.GetServiceURL(service, this.region, this.zone, endpointType) + url, err := this.token.GetServiceURL(service, this.region, this.zone, endpointType) + if err != nil { + url, err = this.client.serviceCatalog.GetServiceURL(service, this.region, this.zone, endpointType) + } + return url, err +} + +func (this *ClientSession) GetServiceURLs(service, endpointType string) ([]string, error) { + if len(this.endpointType) > 0 { + // session specific endpoint type should override the input endpointType, which is supplied by manager + endpointType = this.endpointType + } + if len(this.apiVersion) > 0 && this.apiVersion != DEFAULT_API_VERSION { + service = fmt.Sprintf("%s_%s", service, this.apiVersion) + } + urls, err := this.token.GetServiceURLs(service, this.region, this.zone, endpointType) + if err != nil { + urls, err = this.client.serviceCatalog.GetServiceURLs(service, this.region, this.zone, endpointType) + } + return urls, err } func (this *ClientSession) getBaseUrl(service, endpointType string) (string, error) { diff --git a/pkg/mcclient/token.go b/pkg/mcclient/token.go index 794df976ad..8c0e5e00ed 100644 --- a/pkg/mcclient/token.go +++ b/pkg/mcclient/token.go @@ -11,9 +11,20 @@ type ExternalService struct { Url string } +type Endpoint struct { + Id string + RegionId string + ServiceId string + ServiceName string + Url string + Interface string +} + type TokenCredential interface { gotypes.ISerializable + IServiceCatalog + GetTokenString() string GetDomainId() string GetDomainName() string @@ -30,7 +41,10 @@ type TokenCredential interface { IsAdmin() bool IsSystemAdmin() bool GetRegions() []string - GetServiceURL(service, region, zone, endpointType string) (string, error) + + GetServiceCatalog() IServiceCatalog + GetInternalServices(region string) []string GetExternalServices(region string) []ExternalService + GetEndpoints(region string, endpointType string) []Endpoint } diff --git a/pkg/mcclient/token2.go b/pkg/mcclient/token2.go index 9afd072292..daca173d17 100644 --- a/pkg/mcclient/token2.go +++ b/pkg/mcclient/token2.go @@ -133,7 +133,11 @@ func (this *TokenCredentialV2) IsSystemAdmin() bool { } func (this *TokenCredentialV2) GetServiceURL(service, region, zone, endpointType string) (string, error) { - return this.ServiceCatalog.getServiceURL(service, region, zone, endpointType) + return this.ServiceCatalog.GetServiceURL(service, region, zone, endpointType) +} + +func (this *TokenCredentialV2) GetServiceURLs(service, region, zone, endpointType string) ([]string, error) { + return this.ServiceCatalog.GetServiceURLs(service, region, zone, endpointType) } func (this *TokenCredentialV2) GetInternalServices(region string) []string { @@ -144,6 +148,14 @@ func (this *TokenCredentialV2) GetExternalServices(region string) []ExternalServ return nil } +func (this *TokenCredentialV2) GetEndpoints(region string, endpointType string) []Endpoint { + return nil +} + +func (this *TokenCredentialV2) GetServiceCatalog() IServiceCatalog { + return this.ServiceCatalog +} + func stringArrayContains(arr []string, needle string) bool { for i := 0; i < len(arr); i++ { if arr[i] == needle { @@ -207,7 +219,7 @@ func (catalog KeystoneServiceCatalogV2) getServiceEndpoint(service, region, zone return selected, fmt.Errorf("No such service %s", service) } -func (catalog KeystoneServiceCatalogV2) getServiceURL(service, region, zone, endpointType string) (string, error) { +func (catalog KeystoneServiceCatalogV2) GetServiceURL(service, region, zone, endpointType string) (string, error) { ep, err := catalog.getServiceEndpoint(service, region, zone) if err != nil { return "", err @@ -215,6 +227,14 @@ func (catalog KeystoneServiceCatalogV2) getServiceURL(service, region, zone, end return ep.getURL(endpointType), nil } +func (catalog KeystoneServiceCatalogV2) GetServiceURLs(service, region, zone, endpointType string) ([]string, error) { + url, err := catalog.GetServiceURL(service, region, zone, endpointType) + if err != nil { + return nil, err + } + return []string{url}, nil +} + func (ep KeystoneEndpointV2) getURL(epType string) string { switch epType { case "publicURL": diff --git a/pkg/mcclient/token3.go b/pkg/mcclient/token3.go index 69333f8308..233c336be2 100644 --- a/pkg/mcclient/token3.go +++ b/pkg/mcclient/token3.go @@ -141,7 +141,11 @@ func (this *TokenCredentialV3) GetRegions() []string { } func (this *TokenCredentialV3) GetServiceURL(service, region, zone, endpointType string) (string, error) { - return this.Token.Catalog.getServiceURL(service, region, zone, endpointType) + return this.Token.Catalog.GetServiceURL(service, region, zone, endpointType) +} + +func (this *TokenCredentialV3) GetServiceURLs(service, region, zone, endpointType string) ([]string, error) { + return this.Token.Catalog.GetServiceURLs(service, region, zone, endpointType) } func (this *TokenCredentialV3) GetInternalServices(region string) []string { @@ -152,6 +156,14 @@ func (this *TokenCredentialV3) GetExternalServices(region string) []ExternalServ return this.Token.Catalog.getExternalServices(region) } +func (this *TokenCredentialV3) GetEndpoints(region string, endpointType string) []Endpoint { + return this.Token.Catalog.getEndpoints(region, endpointType) +} + +func (this *TokenCredentialV3) GetServiceCatalog() IServiceCatalog { + return this.Token.Catalog +} + func (catalog KeystoneServiceCatalogV3) getInternalServices(region string) []string { services := make([]string, 0) for i := 0; i < len(catalog); i++ { @@ -200,6 +212,27 @@ func (catalog KeystoneServiceCatalogV3) getRegions() []string { return regions } +func (catalog KeystoneServiceCatalogV3) getEndpoints(region string, endpointType string) []Endpoint { + endpoints := make([]Endpoint, 0) + for i := 0; i < len(catalog); i++ { + for j := 0; j < len(catalog[i].Endpoints); j++ { + endpoint := catalog[i].Endpoints[j] + if (endpoint.Region_id == region || strings.HasPrefix(endpoint.Region_id, region+"-")) && endpoint.Interface == endpointType { + endpoints = append(endpoints, Endpoint{ + endpoint.Id, + endpoint.Region_id, + catalog[i].Id, + catalog[i].Name, + endpoint.Url, + endpoint.Interface, + }) + } + } + } + + return endpoints +} + func RegionID(region, zone string) string { if len(zone) > 0 { return fmt.Sprintf("%s%c%s", region, REGION_ZONE_SEP, zone) @@ -217,7 +250,15 @@ func Id2RegionZone(id string) (string, string) { } } -func (catalog KeystoneServiceCatalogV3) getServiceURL(service, region, zone, endpointType string) (string, error) { +func (catalog KeystoneServiceCatalogV3) GetServiceURL(service, region, zone, endpointType string) (string, error) { + urls, err := catalog.GetServiceURLs(service, region, zone, endpointType) + if err != nil { + return "", err + } + return urls[rand.Intn(len(urls))], nil +} + +func (catalog KeystoneServiceCatalogV3) GetServiceURLs(service, region, zone, endpointType string) ([]string, error) { if endpointType == "" { endpointType = "internalURL" } @@ -248,7 +289,7 @@ func (catalog KeystoneServiceCatalogV3) getServiceURL(service, region, zone, end break } } else { - return "", fmt.Errorf("No default region") + return nil, fmt.Errorf("No default region") } } else { _, ok := regeps[regionzone] @@ -257,14 +298,14 @@ func (catalog KeystoneServiceCatalogV3) getServiceURL(service, region, zone, end } else { selected, ok = regeps[region] if !ok { - return "", fmt.Errorf("No valid %s endpoints for %s in region %s", endpointType, service, RegionID(region, zone)) + return nil, fmt.Errorf("No valid %s endpoints for %s in region %s", endpointType, service, RegionID(region, zone)) } } } - return selected[rand.Intn(len(selected))], nil + return selected, nil } } - return "", fmt.Errorf("No such service %s", service) + return nil, fmt.Errorf("No such service %s", service) } func (self *TokenCredentialV3) String() string { diff --git a/pkg/mcclient/tokensimple.go b/pkg/mcclient/tokensimple.go index 8fbea3a328..5be331fbd9 100644 --- a/pkg/mcclient/tokensimple.go +++ b/pkg/mcclient/tokensimple.go @@ -96,6 +96,10 @@ func (self *SSimpleToken) GetServiceURL(service, region, zone, endpointType stri return "", fmt.Errorf("Not available") } +func (self *SSimpleToken) GetServiceURLs(service, region, zone, endpointType string) ([]string, error) { + return nil, fmt.Errorf("Not available") +} + func (self *SSimpleToken) GetInternalServices(region string) []string { return nil } @@ -104,6 +108,14 @@ func (self *SSimpleToken) GetExternalServices(region string) []ExternalService { return nil } +func (this *SSimpleToken) GetEndpoints(region string, endpointType string) []Endpoint { + return nil +} + +func (this *SSimpleToken) GetServiceCatalog() IServiceCatalog { + return nil +} + func SimplifyToken(token TokenCredential) TokenCredential { simToken, ok := token.(*SSimpleToken) if ok { diff --git a/pkg/scheduler/algorithm/predicates/guest/network_predicate.go b/pkg/scheduler/algorithm/predicates/guest/network_predicate.go index 8f0c13aa50..bd6ece34bf 100644 --- a/pkg/scheduler/algorithm/predicates/guest/network_predicate.go +++ b/pkg/scheduler/algorithm/predicates/guest/network_predicate.go @@ -9,6 +9,7 @@ import ( "yunion.io/x/onecloud/pkg/scheduler/api" "yunion.io/x/onecloud/pkg/scheduler/core" networks "yunion.io/x/onecloud/pkg/scheduler/db/models" + "yunion.io/x/pkg/util/sets" "yunion.io/x/pkg/utils" ) @@ -51,9 +52,9 @@ func (p *NetworkPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor return len(d.HostID) > 0 } - // ServerType's value is 'guest' or ''(support all type) will return true. + // ServerType's value is 'guest', 'container' or ''(support all type) will return true. isMatchServerType := func(network *networks.NetworkSchedResult) bool { - return network.ServerType == "guest" || network.ServerType == "" + return sets.NewString("guest", "", "container").Has(network.ServerType) } counterOfNetwork := func(u *core.Unit, n *networks.NetworkSchedResult, r int) core.Counter { diff --git a/pkg/scheduler/api/sched.go b/pkg/scheduler/api/sched.go index a36ea74ab3..85a7017066 100644 --- a/pkg/scheduler/api/sched.go +++ b/pkg/scheduler/api/sched.go @@ -551,7 +551,15 @@ func newBaremetalDiskConfigFromSimpleJson(sjson *simplejson.Json) (*BaremetalDis rangeArray := make([]int64, 0) ranges := sjson.Get("range").MustArray() for _, size := range ranges { - rangeArray = append(rangeArray, size.(int64)) + sizeNum, ok := size.(json.Number) + if !ok { + return nil, fmt.Errorf("Invalid range size, not json.Number: %v", size) + } + rangeSize, err := sizeNum.Int64() + if err != nil { + return nil, err + } + rangeArray = append(rangeArray, rangeSize) } baremetalDiskConfig.Range = rangeArray diff --git a/pkg/util/aliyun/aliyun.go b/pkg/util/aliyun/aliyun.go index 26b9efce28..9d002a46d1 100644 --- a/pkg/util/aliyun/aliyun.go +++ b/pkg/util/aliyun/aliyun.go @@ -3,6 +3,7 @@ package aliyun import ( "github.com/aliyun/alibaba-cloud-sdk-go/sdk" "github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests" + "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" diff --git a/pkg/util/aliyun/disk.go b/pkg/util/aliyun/disk.go index aeeca8096c..7349b8c7c0 100644 --- a/pkg/util/aliyun/disk.go +++ b/pkg/util/aliyun/disk.go @@ -54,6 +54,16 @@ type SDisk struct { ZoneId string } +func (self *SDisk) GetMetadata() *jsonutils.JSONDict { + data := jsonutils.NewDict() + + // The pricingInfo key structure is 'RegionId::DiskCategory::DiskType + priceKey := fmt.Sprintf("%s::%s::%s", self.RegionId, self.Category, self.Type) + data.Add(jsonutils.NewString(priceKey), "price_key") + + return data +} + func (self *SRegion) GetDisks(instanceId string, zoneId string, category string, diskIds []string, offset int, limit int) ([]SDisk, int, error) { if limit > 50 || limit <= 0 { limit = 50 @@ -138,6 +148,12 @@ func (self *SDisk) Refresh() error { return jsonutils.Update(self, new) } +func (self *SDisk) ResizeDisk(newSize int64) error { + // newSize 单位为 GB. 范围在20 ~2000. 只能往大调。不能调小 + // https://help.aliyun.com/document_detail/25522.html?spm=a2c4g.11174283.6.897.aHwqkS + return self.storage.zone.region.resizeDisk(self.DiskId, newSize) +} + func (self *SDisk) GetDiskFormat() string { return "vhd" } @@ -185,7 +201,7 @@ func (self *SDisk) GetMountpoint() string { return "" } -func (self *SRegion) createDisk(zoneId string, category string, name string, sizeGb int, desc string) (string, error) { +func (self *SRegion) CreateDisk(zoneId string, category string, name string, sizeGb int, desc string) (string, error) { params := make(map[string]string) params["ZoneId"] = zoneId params["DiskName"] = name @@ -223,11 +239,24 @@ func (self *SRegion) deleteDisk(diskId string) error { return err } +func (self *SRegion) DeleteDisk(diskId string) error { + params := make(map[string]string) + params["DiskId"] = diskId + + _, err := self.ecsRequest("DeleteDisk", params) + return err +} + func (self *SRegion) resizeDisk(diskId string, size int64) error { params := make(map[string]string) params["DiskId"] = diskId params["NewSize"] = fmt.Sprintf("%d", size) _, err := self.ecsRequest("ResizeDisk", params) - return err + if err != nil { + log.Errorf("ResizeDisk %s to %s GiB fail %s", diskId, size, err) + return err + } + + return nil } diff --git a/pkg/util/aliyun/errors.go b/pkg/util/aliyun/errors.go new file mode 100644 index 0000000000..da618ab69d --- /dev/null +++ b/pkg/util/aliyun/errors.go @@ -0,0 +1,13 @@ +package aliyun + +import ( + aliyunerrors "github.com/aliyun/alibaba-cloud-sdk-go/sdk/errors" +) + +func isError(err error, code string) bool { + aliyunErr, ok := err.(aliyunerrors.Error) + if ! ok { + return false + } + return aliyunErr.ErrorCode() == code +} diff --git a/pkg/util/aliyun/host.go b/pkg/util/aliyun/host.go index 3ee4a40236..24c5a2a1b1 100644 --- a/pkg/util/aliyun/host.go +++ b/pkg/util/aliyun/host.go @@ -19,6 +19,10 @@ type SHost struct { zone *SZone } +func (self *SHost) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SHost) GetIWires() ([]cloudprovider.ICloudWire, error) { return self.zone.GetIWires() } @@ -157,7 +161,7 @@ func (self *SHost) GetManagerId() string { return self.zone.region.client.providerId } -func (self *SHost) getInstanceById(instanceId string) (*SInstance, error) { +func (self *SHost) GetInstanceById(instanceId string) (*SInstance, error) { inst, err := self.zone.region.GetInstance(instanceId) if err != nil { return nil, err @@ -171,7 +175,7 @@ func (self *SHost) CreateVM(name string, imgId string, sysDiskSize int, cpu int, if err != nil { return nil, err } - vm, err := self.getInstanceById(vmId) + vm, err := self.GetInstanceById(vmId) if err != nil { return nil, err } diff --git a/pkg/util/aliyun/image.go b/pkg/util/aliyun/image.go index 9dfb6f0b3f..849e9eb570 100644 --- a/pkg/util/aliyun/image.go +++ b/pkg/util/aliyun/image.go @@ -55,6 +55,10 @@ type SImage struct { Usage string } +func (self *SImage) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SImage) GetId() string { return self.ImageId } diff --git a/pkg/util/aliyun/instance.go b/pkg/util/aliyun/instance.go index 80a2e9c267..8f4114dc55 100644 --- a/pkg/util/aliyun/instance.go +++ b/pkg/util/aliyun/instance.go @@ -4,6 +4,8 @@ import ( "fmt" "time" + "yunion.io/x/onecloud/pkg/util/seclib2" + "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/util/osprofile" @@ -110,6 +112,7 @@ type SInstance struct { InternetMaxBandwidthIn int InternetMaxBandwidthOut int IoOptimized bool + KeyPairName string Memory int NetworkInterfaces SNetworkInterfaces OSName string @@ -162,6 +165,20 @@ func (self *SRegion) GetInstances(zoneId string, ids []string, offset int, limit return instances, int(total), nil } +func (self *SInstance) GetMetadata() *jsonutils.JSONDict { + data := jsonutils.NewDict() + + // The pricingInfo key structure is 'RegionId::InstanceType::NetworkType::OSType::IoOptimized' + optimized := "optimized" + if !self.IoOptimized { + optimized = "none" + } + priceKey := fmt.Sprintf("%s::%s::%s::%s::%s", self.RegionId, self.InstanceType, self.InstanceNetworkType, self.OSType, optimized) + data.Add(jsonutils.NewString(priceKey), "price_key") + + return data +} + func (self *SInstance) GetCreateTime() time.Time { return self.CreationTime } @@ -322,6 +339,77 @@ func (self *SInstance) GetHypervisor() string { return models.HYPERVISOR_ALIYUN } +func (self *SInstance) StartVM() error { + err := self.host.zone.region.StartVM(self.InstanceId) + if err != nil { + return err + } + return cloudprovider.WaitStatus(self, models.VM_RUNNING, 5*time.Second, 180*time.Second) // 3minutes +} + +func (self *SInstance) StopVM(isForce bool) error { + err := self.host.zone.region.StopVM(self.InstanceId, isForce) + if err != nil { + return err + } + return cloudprovider.WaitStatus(self, models.VM_READY, 10*time.Second, 300*time.Second) // 5mintues +} + +func (self *SInstance) GetVNCInfo() (jsonutils.JSONObject, error) { + url, err := self.host.zone.region.GetInstanceVNCUrl(self.InstanceId) + if err != nil { + return nil, err + } + passwd := seclib.RandomPassword(6) + err = self.host.zone.region.ModifyInstanceVNCUrlPassword(self.InstanceId, passwd) + if err != nil { + return nil, err + } + ret := jsonutils.NewDict() + ret.Add(jsonutils.NewString(url), "url") + ret.Add(jsonutils.NewString(passwd), "password") + ret.Add(jsonutils.NewString("aliyun"), "protocol") + ret.Add(jsonutils.NewString(self.InstanceId), "instance_id") + return ret, nil +} + +func (self *SInstance) UpdateVM(name string) error { + return self.host.zone.region.UpdateVM(self.InstanceId, name) +} + +func (self *SInstance) DeployVM(name string, password string, publicKey string, resetPassword bool, deleteKeypair bool, description string) error { + var keypairName string + if len(publicKey) > 0 { + key, e := self.host.lookUpAliyunKeypair(publicKey) + if e != nil { + key, e = self.host.importAliyunKeypair(publicKey) + if e != nil { + return e + } + } + + keypairName = key + } + + return self.host.zone.region.DeployVM(self.InstanceId, name, password, keypairName, resetPassword, deleteKeypair, description) +} + +func (self *SInstance) RebuildRoot(imageId string) error { + return self.host.zone.region.ReplaceSystemDisk(self.InstanceId, imageId) +} + +func (self *SInstance) ChangeConfig(instanceId string, ncpu int, vmem int) error { + return self.host.zone.region.ChangeVMConfig(self.ZoneId, self.InstanceId, ncpu, vmem, nil) +} + +func (self *SInstance) AttachDisk(diskId string) error { + return self.host.zone.region.AttachDisk(self.InstanceId, diskId) +} + +func (self *SInstance) DetachDisk(diskId string) error { + return self.host.zone.region.DetachDisk(self.InstanceId, diskId) +} + func (self *SRegion) GetInstance(instanceId string) (*SInstance, error) { instances, _, err := self.GetInstances("", []string{instanceId}, 0, 1) if err != nil { @@ -402,7 +490,10 @@ func (self *SRegion) doStopVM(instanceId string, isForce bool) error { } func (self *SRegion) doDeleteVM(instanceId string) error { - return self.instanceOperation(instanceId, "DeleteInstance", nil) + params := make(map[string]string) + params["TerminateSubscription"] = "false" + params["Force"] = "true" + return self.instanceOperation(instanceId, "DeleteInstance", params) } /*func (self *SRegion) waitInstanceStatus(instanceId string, target string, interval time.Duration, timeout time.Duration) error { @@ -425,8 +516,13 @@ func (self *SInstance) waitStatus(target string, interval time.Duration, timeout }*/ func (self *SRegion) StartVM(instanceId string) error { - status, _ := self.GetInstanceStatus(instanceId) + status, err := self.GetInstanceStatus(instanceId) + if err != nil { + log.Errorf("Fail to get instance status on StartVM: %s", err) + return err + } if status != InstanceStatusStopped { + log.Errorf("StartVM: vm status is %s expect %s", status, InstanceStatusStopped) return cloudprovider.ErrInvalidStatus } return self.doStartVM(instanceId) @@ -437,8 +533,13 @@ func (self *SRegion) StartVM(instanceId string) error { } func (self *SRegion) StopVM(instanceId string, isForce bool) error { - status, _ := self.GetInstanceStatus(instanceId) + status, err := self.GetInstanceStatus(instanceId) + if err != nil { + log.Errorf("Fail to get instance status on StopVM: %s", err) + return err + } if status != InstanceStatusRunning { + log.Errorf("StopVM: vm status is %s expect %s", status, InstanceStatusRunning) return cloudprovider.ErrInvalidStatus } return self.doStopVM(instanceId, isForce) @@ -450,13 +551,13 @@ func (self *SRegion) StopVM(instanceId string, isForce bool) error { func (self *SRegion) DeleteVM(instanceId string) error { status, err := self.GetInstanceStatus(instanceId) - if status == InstanceStatusRunning { - err = self.StopVM(instanceId, true) - if err != nil { - return err - } - } else if status != InstanceStatusStopped { - return cloudprovider.ErrInvalidStatus + if err != nil { + log.Errorf("Fail to get instance status on DeleteVM: %s", err) + return err + } + log.Debugf("Instance status on delete is %s", status) + if status != InstanceStatusStopped { + log.Warningf("DeleteVM: vm status is %s expect %s", status, InstanceStatusStopped) } return self.doDeleteVM(instanceId) // if err != nil { @@ -472,46 +573,128 @@ func (self *SRegion) DeleteVM(instanceId string) error { // } } -func (self *SInstance) StartVM() error { - err := self.host.zone.region.StartVM(self.InstanceId) +func (self *SRegion) DeployVM(instanceId string, name string, password string, keypairName string, resetPassword bool, deleteKeypair bool, description string) error { + instance, err := self.GetInstance(instanceId) if err != nil { return err } - return cloudprovider.WaitStatus(self, models.VM_RUNNING, 5*time.Second, 180*time.Second) // 3minutes -} -func (self *SInstance) StopVM(isForce bool) error { - err := self.host.zone.region.StopVM(self.InstanceId, isForce) - if err != nil { - return err + // 修改密钥时直接返回 + if deleteKeypair { + return self.DetachKeyPair(instanceId, instance.KeyPairName) + } + + if len(keypairName) > 0 { + return self.AttachKeypair(instanceId, keypairName) + } + + params := make(map[string]string) + if resetPassword { + params["Password"] = seclib2.RandomPassword2(12) + } + // 指定密码的情况下,使用指定的密码 + if len(password) > 0 { + params["Password"] = password + } + + if len(name) > 0 && instance.InstanceName != name { + params["InstanceName"] = name + params["HostName"] = name + } + + if len(description) > 0 && instance.Description != description { + params["Description"] = description + } + + if len(params) > 0 { + return self.modifyInstanceAttribute(instanceId, params) + } else { + return nil } - return cloudprovider.WaitStatus(self, models.VM_READY, 10*time.Second, 300*time.Second) // 5mintues } func (self *SInstance) DeleteVM() error { - err := self.host.zone.region.DeleteVM(self.InstanceId) - if err != nil { - return err + for { + err := self.host.zone.region.DeleteVM(self.InstanceId) + if err != nil { + if isError(err, "IncorrectInstanceStatus.Initializing") { + log.Infof("The instance is initializing, try later ...") + time.Sleep(10 * time.Second) + } else { + return err + } + } else { + break + } } return cloudprovider.WaitDeleted(self, 10*time.Second, 300*time.Second) // 5minutes } -func (self *SInstance) GetVNCInfo() (jsonutils.JSONObject, error) { - url, err := self.host.zone.region.GetInstanceVNCUrl(self.InstanceId) - if err != nil { - return nil, err +func (self *SRegion) UpdateVM(instanceId string, hostname string) error { + /* + api: ModifyInstanceAttribute + https://help.aliyun.com/document_detail/25503.html?spm=a2c4g.11186623.4.1.DrgpjW + */ + params := make(map[string]string) + params["HostName"] = hostname + return self.modifyInstanceAttribute(instanceId, params) +} + +func (self *SRegion) modifyInstanceAttribute(instanceId string, params map[string]string) error { + return self.instanceOperation(instanceId, "ModifyInstanceAttribute", params) +} + +func (self *SRegion) ReplaceSystemDisk(instanceId string, image string) error { + params := make(map[string]string) + params["ImageId"] = image + return self.instanceOperation(instanceId, "ReplaceSystemDisk", params) +} + +func (self *SRegion) ChangeVMConfig(zoneId string, instanceId string, ncpu int, vmem int, disks []*SDisk) error { + // todo: support change disk config? + params := make(map[string]string) + instanceTypes, e := self.GetMatchInstanceTypes(ncpu, vmem, 0, zoneId) + if e != nil { + return e } - passwd := seclib.RandomPassword(6) - err = self.host.zone.region.ModifyInstanceVNCUrlPassword(self.InstanceId, passwd) - if err != nil { - return nil, err + + for _, instancetype := range instanceTypes { + params["InstanceType"] = instancetype.InstanceTypeId + params["ClientToken"] = utils.GenRequestId(20) + if err := self.instanceOperation(instanceId, "ModifyInstanceSpec", params); err != nil { + log.Errorf("Failed for %s: %s", instancetype.InstanceTypeId, err) + } else { + return nil + } } - ret := jsonutils.NewDict() - ret.Add(jsonutils.NewString(url), "url") - ret.Add(jsonutils.NewString(passwd), "password") - ret.Add(jsonutils.NewString("aliyun"), "protocol") - ret.Add(jsonutils.NewString(self.InstanceId), "instance_id") - return ret, nil + + return fmt.Errorf("Failed to change vm config, specification not supported") +} + +func (self *SRegion) DetachDisk(instanceId string, diskId string) error { + params := make(map[string]string) + params["InstanceId"] = instanceId + params["DiskId"] = diskId + _, err := self.ecsRequest("DetachDisk", params) + if err != nil { + log.Errorf("DetachDisk %s to %s fail %s", diskId, instanceId, err) + return err + } + + return nil +} + +func (self *SRegion) AttachDisk(instanceId string, diskId string) error { + params := make(map[string]string) + params["InstanceId"] = instanceId + params["DiskId"] = diskId + _, err := self.ecsRequest("AttachDisk", params) + if err != nil { + log.Errorf("AttachDisk %s to %s fail %s", diskId, instanceId, err) + return err + } + + return nil } func (self *SInstance) SyncSecurityGroup(secgroupId string, name string, rules []secrules.SecurityRule) error { diff --git a/pkg/util/aliyun/keypair.go b/pkg/util/aliyun/keypair.go index c8da45b6a0..e3b2a504a8 100644 --- a/pkg/util/aliyun/keypair.go +++ b/pkg/util/aliyun/keypair.go @@ -1,6 +1,7 @@ package aliyun import ( + "encoding/json" "fmt" "yunion.io/x/log" ) @@ -62,3 +63,33 @@ func (self *SRegion) ImportKeypair(name string, pubKey string) (*SKeypair, error } return &keypair, nil } + +func (self *SRegion) AttachKeypair(instanceId string, name string) error { + params := make(map[string]string) + params["RegionId"] = self.RegionId + params["KeyPairName"] = name + instances, _ := json.Marshal(&[...]string{instanceId}) + params["InstanceIds"] = string(instances) + _, err := self.ecsRequest("AttachKeyPair", params) + if err != nil { + log.Errorf("AttachKeyPair fail %s", err) + return err + } + + return nil +} + +func (self *SRegion) DetachKeyPair(instanceId string, name string) error { + params := make(map[string]string) + params["RegionId"] = self.RegionId + params["KeyPairName"] = name + instances, _ := json.Marshal(&[...]string{instanceId}) + params["InstanceIds"] = string(instances) + _, err := self.ecsRequest("DetachKeyPair", params) + if err != nil { + log.Errorf("DetachKeyPair fail %s", err) + return err + } + + return nil +} diff --git a/pkg/util/aliyun/region.go b/pkg/util/aliyun/region.go index db08f85c25..311f7bf195 100644 --- a/pkg/util/aliyun/region.go +++ b/pkg/util/aliyun/region.go @@ -9,9 +9,10 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/utils" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" - "yunion.io/x/pkg/utils" ) type SRegion struct { @@ -35,6 +36,10 @@ func (self *SRegion) GetClient() *SAliyunClient { return self.client } +func (self *SRegion) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SRegion) getEcsClient() (*sdk.Client, error) { if self.ecsClient == nil { cli, err := sdk.NewClientWithAccessKey(self.RegionId, self.client.accessKey, self.client.secret) @@ -564,4 +569,4 @@ func (self *SRegion) updateInstance(instId string, name, desc, passwd, hostname func (self *SRegion) UpdateInstancePassword(instId string, passwd string) error { return self.updateInstance(instId, "", "", passwd, "") -} \ No newline at end of file +} diff --git a/pkg/util/aliyun/securitygroup.go b/pkg/util/aliyun/securitygroup.go index 69515ae79d..3082451737 100644 --- a/pkg/util/aliyun/securitygroup.go +++ b/pkg/util/aliyun/securitygroup.go @@ -77,6 +77,10 @@ func (v PermissionSet) Less(i, j int) bool { return false } +func (self *SSecurityGroup) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SSecurityGroup) GetId() string { return self.SecurityGroupId } @@ -574,3 +578,15 @@ func (self *SRegion) leaveSecurityGroup(secgroupId, instanceId string) error { _, err := self.ecsRequest("LeaveSecurityGroup", params) return err } + +func (self *SRegion) deleteSecurityGroup(secGrpId string) error { + params := make(map[string]string) + params["SecurityGroupId"] = secGrpId + + _, err := self.ecsRequest("DeleteSecurityGroup", params) + if err != nil { + log.Errorf("Delete security group fail %s", err) + return err + } + return nil +} diff --git a/pkg/util/aliyun/shell/disk.go b/pkg/util/aliyun/shell/disk.go index 1196bb5739..19cc4c7693 100644 --- a/pkg/util/aliyun/shell/disk.go +++ b/pkg/util/aliyun/shell/disk.go @@ -21,4 +21,15 @@ func init() { printList(disks, total, args.Offset, args.Limit, []string{}) return nil }) + + type DiskDeleteOptions struct { + Instance string `help:"Instance ID"` + } + shellutils.R(&DiskDeleteOptions{}, "disk-delete", "List disks", func(cli *aliyun.SRegion, args *DiskDeleteOptions) error { + e := cli.DeleteDisk(args.Instance) + if e != nil { + return e + } + return nil + }) } diff --git a/pkg/util/aliyun/shell/instance.go b/pkg/util/aliyun/shell/instance.go index 9c86e47aa8..4a741d0b66 100644 --- a/pkg/util/aliyun/shell/instance.go +++ b/pkg/util/aliyun/shell/instance.go @@ -2,6 +2,7 @@ package shell import ( "fmt" + "yunion.io/x/onecloud/pkg/util/aliyun" "yunion.io/x/onecloud/pkg/util/shellutils" ) @@ -81,8 +82,65 @@ func init() { return nil }) + /* + server-change-config 更改系统配置 + server-reset + */ + type InstanceDeployOptions struct { + ID string `help:"instance ID"` + Name string `help:"new instance name"` + Hostname string `help:"new hostname"` + Keypair string `help:"Keypair Name"` + DeleteKeypair bool `help:"Remove SSH keypair"` + Password string `help:"new password"` + ResetPassword bool `help:"Force reset password"` + Description string `help:"new instances description"` + } + + shellutils.R(&InstanceDeployOptions{}, "instance-deploy", "Deploy keypair/password to a stopped virtual server", func(cli *aliyun.SRegion, args *InstanceDeployOptions) error { + err := cli.DeployVM(args.ID, args.Name, args.Password, args.Keypair, args.ResetPassword, args.DeleteKeypair, args.Description) + if err != nil { + return err + } + return nil + }) + + type InstanceRebuildRootOptions struct { + ID string `help:"instance ID"` + Image string `help:"Image ID"` + } + + shellutils.R(&InstanceRebuildRootOptions{}, "instance-rebuild-root", "Reinstall virtual server system image", func(cli *aliyun.SRegion, args *InstanceRebuildRootOptions) error { + err := cli.ReplaceSystemDisk(args.ID, args.Image) + if err != nil { + return err + } + return nil + }) + + type InstanceChangeConfigOptions struct { + ID string `help:"instance ID"` + Ncpu int `help:"number of CPU"` + Vmem int `help:"MiB of memory"` + Disk []int `help:"Data disk sizes int GB"` + } + + shellutils.R(&InstanceChangeConfigOptions{}, "instance-change-config", "Deploy keypair/password to a stopped virtual server", func(cli *aliyun.SRegion, args *InstanceChangeConfigOptions) error { + instance, e := cli.GetInstance(args.ID) + if e != nil { + return e + } + + // todo : add create disks + err := cli.ChangeVMConfig(instance.ZoneId, args.ID, args.Ncpu, args.Vmem, nil) + if err != nil { + return err + } + return nil + }) + type InstanceUpdatePasswordOptions struct { - ID string `help:"Instance ID"` + ID string `help:"Instance ID"` PASSWD string `help:"new password"` } shellutils.R(&InstanceUpdatePasswordOptions{}, "instance-update-password", "Update instance password", func(cli *aliyun.SRegion, args *InstanceUpdatePasswordOptions) error { diff --git a/pkg/util/aliyun/storage.go b/pkg/util/aliyun/storage.go index b54700b1e6..d8dec31739 100644 --- a/pkg/util/aliyun/storage.go +++ b/pkg/util/aliyun/storage.go @@ -15,6 +15,10 @@ type SStorage struct { storageType string } +func (self *SStorage) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SStorage) GetId() string { return fmt.Sprintf("%s-%s-%s", self.zone.region.client.providerId, self.zone.GetId(), self.storageType) } @@ -100,7 +104,7 @@ func (self *SStorage) GetIStoragecache() cloudprovider.ICloudStoragecache { } func (self *SStorage) CreateIDisk(name string, sizeGb int, desc string) (cloudprovider.ICloudDisk, error) { - diskId, err := self.zone.region.createDisk(self.zone.ZoneId, self.storageType, name, sizeGb, desc) + diskId, err := self.zone.region.CreateDisk(self.zone.ZoneId, self.storageType, name, sizeGb, desc) if err != nil { log.Errorf("createDisk fail %s", err) return nil, err diff --git a/pkg/util/aliyun/storagecache.go b/pkg/util/aliyun/storagecache.go index 61394948c2..fab2072253 100644 --- a/pkg/util/aliyun/storagecache.go +++ b/pkg/util/aliyun/storagecache.go @@ -5,6 +5,7 @@ import ( "strings" "time" + "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/options" @@ -19,6 +20,10 @@ type SStoragecache struct { iimages []cloudprovider.ICloudImage } +func (self *SStoragecache) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SStoragecache) GetId() string { return fmt.Sprintf("%s-%s", self.region.client.providerId, self.region.GetId()) } diff --git a/pkg/util/aliyun/vpc.go b/pkg/util/aliyun/vpc.go index 9751039c39..ea645ff2be 100644 --- a/pkg/util/aliyun/vpc.go +++ b/pkg/util/aliyun/vpc.go @@ -45,6 +45,10 @@ type SVpc struct { VpcName string } +func (self *SVpc) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SVpc) GetId() string { return self.VpcId } @@ -184,6 +188,19 @@ func (self *SVpc) GetManagerId() string { } func (self *SVpc) Delete() error { + err := self.fetchSecurityGroups() + if err != nil { + log.Errorf("fetchSecurityGroup for VPC delete fail %s", err) + return err + } + for i := 0; i < len(self.secgroups); i += 1 { + secgroup := self.secgroups[i].(*SSecurityGroup) + err := self.region.deleteSecurityGroup(secgroup.SecurityGroupId) + if err != nil { + log.Errorf("deleteSecurityGroup for VPC delete fail %s", err) + return err + } + } return self.region.DeleteVpc(self.VpcId) } diff --git a/pkg/util/aliyun/vswitch.go b/pkg/util/aliyun/vswitch.go index 9c9e7bbe84..bef2c3d74d 100644 --- a/pkg/util/aliyun/vswitch.go +++ b/pkg/util/aliyun/vswitch.go @@ -34,6 +34,10 @@ type SVSwitch struct { ZoneId string } +func (self *SVSwitch) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SVSwitch) GetId() string { return self.VSwitchId } diff --git a/pkg/util/aliyun/wire.go b/pkg/util/aliyun/wire.go index 903dcc681a..12793359c1 100644 --- a/pkg/util/aliyun/wire.go +++ b/pkg/util/aliyun/wire.go @@ -3,6 +3,7 @@ package aliyun import ( "fmt" + "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" ) @@ -14,6 +15,10 @@ type SWire struct { inetworks []cloudprovider.ICloudNetwork } +func (self *SWire) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SWire) GetId() string { return fmt.Sprintf("%s-%s", self.vpc.GetId(), self.zone.GetId()) } diff --git a/pkg/util/aliyun/zone.go b/pkg/util/aliyun/zone.go index 844b4d6e6f..2241e4d9ce 100644 --- a/pkg/util/aliyun/zone.go +++ b/pkg/util/aliyun/zone.go @@ -3,6 +3,7 @@ package aliyun import ( "fmt" + "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" @@ -111,6 +112,10 @@ type SZone struct { AvailableDedicatedHostTypes SDedicatedHostTypes } +func (self *SZone) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SZone) GetId() string { return self.ZoneId } diff --git a/pkg/util/esxi/datacenter.go b/pkg/util/esxi/datacenter.go index 7da30e1dfb..6c9d1c6d0d 100644 --- a/pkg/util/esxi/datacenter.go +++ b/pkg/util/esxi/datacenter.go @@ -76,3 +76,4 @@ func (dc *SDatacenter) GetIStorages() ([]cloudprovider.ICloudStorage, error) { } return dc.istorages, nil } + diff --git a/pkg/util/esxi/host.go b/pkg/util/esxi/host.go index 4f524948d2..3a9a053c5d 100644 --- a/pkg/util/esxi/host.go +++ b/pkg/util/esxi/host.go @@ -68,6 +68,10 @@ func NewHost(manager *SESXiClient, host *mo.HostSystem, dc *SDatacenter) *SHost return &SHost{SManagedObject: newManagedObject(manager, host, dc)} } +func (self *SHost) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SHost) getHostSystem() *mo.HostSystem { return self.object.(*mo.HostSystem) } diff --git a/pkg/util/esxi/manager.go b/pkg/util/esxi/manager.go index 0c0698bfcd..ed1dd5d76d 100644 --- a/pkg/util/esxi/manager.go +++ b/pkg/util/esxi/manager.go @@ -5,10 +5,12 @@ import ( "fmt" "net/url" "reflect" + "strings" "github.com/vmware/govmomi" "github.com/vmware/govmomi/object" "github.com/vmware/govmomi/property" + "github.com/vmware/govmomi/session" "github.com/vmware/govmomi/view" "github.com/vmware/govmomi/vim25/mo" "github.com/vmware/govmomi/vim25/types" @@ -16,10 +18,8 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - "github.com/vmware/govmomi/session" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" - "strings" ) const ( diff --git a/pkg/util/esxi/mobase.go b/pkg/util/esxi/mobase.go index 23f6aa9c61..c60498ebc5 100644 --- a/pkg/util/esxi/mobase.go +++ b/pkg/util/esxi/mobase.go @@ -1,10 +1,13 @@ package esxi import ( - "github.com/vmware/govmomi/vim25/mo" - "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudprovider" "reflect" + + "github.com/vmware/govmomi/vim25/mo" + + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/cloudprovider" ) type SManagedObject struct { diff --git a/pkg/util/esxi/storage.go b/pkg/util/esxi/storage.go index 90846ed5a9..b64c57ce1f 100644 --- a/pkg/util/esxi/storage.go +++ b/pkg/util/esxi/storage.go @@ -18,6 +18,10 @@ func NewDatastore(manager *SESXiClient, ds *mo.Datastore, dc *SDatacenter) *SDat return &SDatastore{SManagedObject: newManagedObject(manager, ds, dc)} } +func (self *SDatastore) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SDatastore) getDatastore() *mo.Datastore { return self.object.(*mo.Datastore) } diff --git a/pkg/util/esxi/virtualmachine.go b/pkg/util/esxi/virtualmachine.go index 2005bbc9bc..5cf0149ee9 100644 --- a/pkg/util/esxi/virtualmachine.go +++ b/pkg/util/esxi/virtualmachine.go @@ -27,6 +27,10 @@ func NewVirtualMachine(manager *SESXiClient, vm *mo.VirtualMachine, dc *SDatacen return &SVirtualMachine{SManagedObject: newManagedObject(manager, vm, dc), host: host} } +func (self *SVirtualMachine) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SVirtualMachine) getVirtualMachine() *mo.VirtualMachine { return self.object.(*mo.VirtualMachine) } @@ -65,6 +69,26 @@ func (self *SVirtualMachine) IsEmulated() bool { return false } +func (self *SVirtualMachine) DeployVM(name string, password string, publicKey string, resetPassword bool, deleteKeypair bool, description string) error { + return cloudprovider.ErrNotImplemented +} + +func (self *SVirtualMachine) RebuildRoot(imageId string) error { + return cloudprovider.ErrNotImplemented +} + +func (self *SVirtualMachine) UpdateVM(name string) error { + return cloudprovider.ErrNotImplemented +} + +func (self *SVirtualMachine) DetachDisk(diskId string) error { + return cloudprovider.ErrNotImplemented +} + +func (self *SVirtualMachine) AttachDisk(diskId string) error { + return cloudprovider.ErrNotImplemented +} + func (self *SVirtualMachine) getUuid() string { return self.getVirtualMachine().Summary.Config.Uuid } @@ -209,3 +233,7 @@ func (self *SVirtualMachine) acquireVmrcUrl() (jsonutils.JSONObject, error) { ret.Add(jsonutils.NewString(url), "url") return ret, nil } + +func (dc *SVirtualMachine) ChangeConfig(instanceId string, ncpu int, vmem int) error { + return cloudprovider.ErrNotImplemented +} diff --git a/pkg/util/printutils/printjson.go b/pkg/util/printutils/printjson.go index ed01e9ce4e..da14f48590 100644 --- a/pkg/util/printutils/printjson.go +++ b/pkg/util/printutils/printjson.go @@ -58,7 +58,7 @@ func PrintJSONList(list *modules.ListResult, columns []string) { } rows = append(rows, row) } - fmt.Println(pt.GetString(rows)) + fmt.Print(pt.GetString(rows)) title := fmt.Sprintf("Total: %d", list.Total) if list.Limit == 0 && list.Total > len(list.Data) { list.Limit = len(list.Data) @@ -95,7 +95,7 @@ func PrintJSONObject(obj jsonutils.JSONObject) { } rows = append(rows, row) } - fmt.Println(pt.GetString(rows)) + fmt.Print(pt.GetString(rows)) } func PrintJSONBatchResults(results []modules.SubmitResult, columns []string) { diff --git a/pkg/util/seclib2/seclib.go b/pkg/util/seclib2/seclib.go index 86cee26422..d3fd831df3 100644 --- a/pkg/util/seclib2/seclib.go +++ b/pkg/util/seclib2/seclib.go @@ -1,10 +1,10 @@ package seclib2 import ( - "fmt" - "strings" "bytes" + "fmt" "math/rand" + "strings" ) const ( @@ -39,7 +39,7 @@ func RandomPassword2(width int) string { } buf.WriteByte(ch) } - if digitsCnt > 1 && letterCnt > 1 && upperCnt > 1 && puncCnt >=1 && puncCnt <= 2 { + if digitsCnt > 1 && letterCnt > 1 && upperCnt > 1 && puncCnt >= 1 && puncCnt <= 2 { return buf.String() } } diff --git a/vendor/github.com/aliyun/alibaba-cloud-sdk-go/services/push/push_notice_to_android.go b/vendor/github.com/aliyun/alibaba-cloud-sdk-go/services/push/push_notice_to_android.go deleted file mode 100644 index ef385f3a9f..0000000000 --- a/vendor/github.com/aliyun/alibaba-cloud-sdk-go/services/push/push_notice_to_android.go +++ /dev/null @@ -1,110 +0,0 @@ -package push - -//Licensed under the Apache License, Version 2.0 (the "License"); -//you may not use this file except in compliance with the License. -//You may obtain a copy of the License at -// -//http://www.apache.org/licenses/LICENSE-2.0 -// -//Unless required by applicable law or agreed to in writing, software -//distributed under the License is distributed on an "AS IS" BASIS, -//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -//See the License for the specific language governing permissions and -//limitations under the License. -// -// Code generated by Alibaba Cloud SDK Code Generator. -// Changes may cause incorrect behavior and will be lost if the code is regenerated. - -import ( - "github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests" - "github.com/aliyun/alibaba-cloud-sdk-go/sdk/responses" -) - -// PushNoticeToAndroid invokes the push.PushNoticeToAndroid API synchronously -// api document: https://help.aliyun.com/api/push/pushnoticetoandroid.html -func (client *Client) PushNoticeToAndroid(request *PushNoticeToAndroidRequest) (response *PushNoticeToAndroidResponse, err error) { - response = CreatePushNoticeToAndroidResponse() - err = client.DoAction(request, response) - return -} - -// PushNoticeToAndroidWithChan invokes the push.PushNoticeToAndroid API asynchronously -// api document: https://help.aliyun.com/api/push/pushnoticetoandroid.html -// asynchronous document: https://help.aliyun.com/document_detail/66220.html -func (client *Client) PushNoticeToAndroidWithChan(request *PushNoticeToAndroidRequest) (<-chan *PushNoticeToAndroidResponse, <-chan error) { - responseChan := make(chan *PushNoticeToAndroidResponse, 1) - errChan := make(chan error, 1) - err := client.AddAsyncTask(func() { - defer close(responseChan) - defer close(errChan) - response, err := client.PushNoticeToAndroid(request) - if err != nil { - errChan <- err - } else { - responseChan <- response - } - }) - if err != nil { - errChan <- err - close(responseChan) - close(errChan) - } - return responseChan, errChan -} - -// PushNoticeToAndroidWithCallback invokes the push.PushNoticeToAndroid API asynchronously -// api document: https://help.aliyun.com/api/push/pushnoticetoandroid.html -// asynchronous document: https://help.aliyun.com/document_detail/66220.html -func (client *Client) PushNoticeToAndroidWithCallback(request *PushNoticeToAndroidRequest, callback func(response *PushNoticeToAndroidResponse, err error)) <-chan int { - result := make(chan int, 1) - err := client.AddAsyncTask(func() { - var response *PushNoticeToAndroidResponse - var err error - defer close(result) - response, err = client.PushNoticeToAndroid(request) - callback(response, err) - result <- 1 - }) - if err != nil { - defer close(result) - callback(nil, err) - result <- 0 - } - return result -} - -// PushNoticeToAndroidRequest is the request struct for api PushNoticeToAndroid -type PushNoticeToAndroidRequest struct { - *requests.RpcRequest - AppKey requests.Integer `position:"Query" name:"AppKey"` - Target string `position:"Query" name:"Target"` - TargetValue string `position:"Query" name:"TargetValue"` - Title string `position:"Query" name:"Title"` - Body string `position:"Query" name:"Body"` - JobKey string `position:"Query" name:"JobKey"` - ExtParameters string `position:"Query" name:"ExtParameters"` -} - -// PushNoticeToAndroidResponse is the response struct for api PushNoticeToAndroid -type PushNoticeToAndroidResponse struct { - *responses.BaseResponse - RequestId string `json:"RequestId" xml:"RequestId"` - MessageId string `json:"MessageId" xml:"MessageId"` -} - -// CreatePushNoticeToAndroidRequest creates a request to invoke PushNoticeToAndroid API -func CreatePushNoticeToAndroidRequest() (request *PushNoticeToAndroidRequest) { - request = &PushNoticeToAndroidRequest{ - RpcRequest: &requests.RpcRequest{}, - } - request.InitWithApiInfo("Push", "2016-08-01", "PushNoticeToAndroid", "", "") - return -} - -// CreatePushNoticeToAndroidResponse creates a response to parse from PushNoticeToAndroid response -func CreatePushNoticeToAndroidResponse() (response *PushNoticeToAndroidResponse) { - response = &PushNoticeToAndroidResponse{ - BaseResponse: &responses.BaseResponse{}, - } - return -} diff --git a/vendor/github.com/aliyun/alibaba-cloud-sdk-go/services/push/push_notice_toi_os.go b/vendor/github.com/aliyun/alibaba-cloud-sdk-go/services/push/push_notice_toi_os.go deleted file mode 100644 index 73d1a588e6..0000000000 --- a/vendor/github.com/aliyun/alibaba-cloud-sdk-go/services/push/push_notice_toi_os.go +++ /dev/null @@ -1,111 +0,0 @@ -package push - -//Licensed under the Apache License, Version 2.0 (the "License"); -//you may not use this file except in compliance with the License. -//You may obtain a copy of the License at -// -//http://www.apache.org/licenses/LICENSE-2.0 -// -//Unless required by applicable law or agreed to in writing, software -//distributed under the License is distributed on an "AS IS" BASIS, -//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -//See the License for the specific language governing permissions and -//limitations under the License. -// -// Code generated by Alibaba Cloud SDK Code Generator. -// Changes may cause incorrect behavior and will be lost if the code is regenerated. - -import ( - "github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests" - "github.com/aliyun/alibaba-cloud-sdk-go/sdk/responses" -) - -// PushNoticeToiOS invokes the push.PushNoticeToiOS API synchronously -// api document: https://help.aliyun.com/api/push/pushnoticetoios.html -func (client *Client) PushNoticeToiOS(request *PushNoticeToiOSRequest) (response *PushNoticeToiOSResponse, err error) { - response = CreatePushNoticeToiOSResponse() - err = client.DoAction(request, response) - return -} - -// PushNoticeToiOSWithChan invokes the push.PushNoticeToiOS API asynchronously -// api document: https://help.aliyun.com/api/push/pushnoticetoios.html -// asynchronous document: https://help.aliyun.com/document_detail/66220.html -func (client *Client) PushNoticeToiOSWithChan(request *PushNoticeToiOSRequest) (<-chan *PushNoticeToiOSResponse, <-chan error) { - responseChan := make(chan *PushNoticeToiOSResponse, 1) - errChan := make(chan error, 1) - err := client.AddAsyncTask(func() { - defer close(responseChan) - defer close(errChan) - response, err := client.PushNoticeToiOS(request) - if err != nil { - errChan <- err - } else { - responseChan <- response - } - }) - if err != nil { - errChan <- err - close(responseChan) - close(errChan) - } - return responseChan, errChan -} - -// PushNoticeToiOSWithCallback invokes the push.PushNoticeToiOS API asynchronously -// api document: https://help.aliyun.com/api/push/pushnoticetoios.html -// asynchronous document: https://help.aliyun.com/document_detail/66220.html -func (client *Client) PushNoticeToiOSWithCallback(request *PushNoticeToiOSRequest, callback func(response *PushNoticeToiOSResponse, err error)) <-chan int { - result := make(chan int, 1) - err := client.AddAsyncTask(func() { - var response *PushNoticeToiOSResponse - var err error - defer close(result) - response, err = client.PushNoticeToiOS(request) - callback(response, err) - result <- 1 - }) - if err != nil { - defer close(result) - callback(nil, err) - result <- 0 - } - return result -} - -// PushNoticeToiOSRequest is the request struct for api PushNoticeToiOS -type PushNoticeToiOSRequest struct { - *requests.RpcRequest - AppKey requests.Integer `position:"Query" name:"AppKey"` - Target string `position:"Query" name:"Target"` - TargetValue string `position:"Query" name:"TargetValue"` - ApnsEnv string `position:"Query" name:"ApnsEnv"` - Title string `position:"Query" name:"Title"` - Body string `position:"Query" name:"Body"` - JobKey string `position:"Query" name:"JobKey"` - ExtParameters string `position:"Query" name:"ExtParameters"` -} - -// PushNoticeToiOSResponse is the response struct for api PushNoticeToiOS -type PushNoticeToiOSResponse struct { - *responses.BaseResponse - RequestId string `json:"RequestId" xml:"RequestId"` - MessageId string `json:"MessageId" xml:"MessageId"` -} - -// CreatePushNoticeToiOSRequest creates a request to invoke PushNoticeToiOS API -func CreatePushNoticeToiOSRequest() (request *PushNoticeToiOSRequest) { - request = &PushNoticeToiOSRequest{ - RpcRequest: &requests.RpcRequest{}, - } - request.InitWithApiInfo("Push", "2016-08-01", "PushNoticeToiOS", "", "") - return -} - -// CreatePushNoticeToiOSResponse creates a response to parse from PushNoticeToiOS response -func CreatePushNoticeToiOSResponse() (response *PushNoticeToiOSResponse) { - response = &PushNoticeToiOSResponse{ - BaseResponse: &responses.BaseResponse{}, - } - return -} diff --git a/vendor/github.com/gosuri/uitable/.travis.yml b/vendor/github.com/gosuri/uitable/.travis.yml new file mode 100644 index 0000000000..2139b89ef0 --- /dev/null +++ b/vendor/github.com/gosuri/uitable/.travis.yml @@ -0,0 +1,8 @@ +language: go +sudo: false +install: + - go get ./... +go: + - 1.4 + - 1.5.2 + - tip diff --git a/vendor/github.com/gosuri/uitable/LICENSE b/vendor/github.com/gosuri/uitable/LICENSE new file mode 100644 index 0000000000..e436d90439 --- /dev/null +++ b/vendor/github.com/gosuri/uitable/LICENSE @@ -0,0 +1,10 @@ +MIT License +=========== + +Copyright (c) 2015, Greg Osuri + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/gosuri/uitable/Makefile b/vendor/github.com/gosuri/uitable/Makefile new file mode 100644 index 0000000000..60246a8a93 --- /dev/null +++ b/vendor/github.com/gosuri/uitable/Makefile @@ -0,0 +1,4 @@ +test: + @go test ./... + +.PHONY: test diff --git a/vendor/github.com/gosuri/uitable/README.md b/vendor/github.com/gosuri/uitable/README.md new file mode 100644 index 0000000000..683b538d13 --- /dev/null +++ b/vendor/github.com/gosuri/uitable/README.md @@ -0,0 +1,67 @@ +# uitable [![GoDoc](https://godoc.org/github.com/gosuri/uitable?status.svg)](https://godoc.org/github.com/gosuri/uitable) [![Build Status](https://travis-ci.org/gosuri/uitable.svg?branch=master)](https://travis-ci.org/gosuri/uitable) + +uitable is a go library for representing data as tables for terminal applications. It provides primitives for sizing and wrapping columns to improve readability. + +## Example Usage + +Full source code for the example is available at [example/main.go](example/main.go) + +```go +table := uitable.New() +table.MaxColWidth = 50 + +table.AddRow("NAME", "BIRTHDAY", "BIO") +for _, hacker := range hackers { + table.AddRow(hacker.Name, hacker.Birthday, hacker.Bio) +} +fmt.Println(table) +``` + +Will render the data as: + +```sh +NAME BIRTHDAY BIO +Ada Lovelace December 10, 1815 Ada was a British mathematician and writer, chi... +Alan Turing June 23, 1912 Alan was a British pioneering computer scientis... +``` + +For wrapping in two columns: + +```go +table = uitable.New() +table.MaxColWidth = 80 +table.Wrap = true // wrap columns + +for _, hacker := range hackers { + table.AddRow("Name:", hacker.Name) + table.AddRow("Birthday:", hacker.Birthday) + table.AddRow("Bio:", hacker.Bio) + table.AddRow("") // blank +} +fmt.Println(table) +``` + +Will render the data as: + +``` +Name: Ada Lovelace +Birthday: December 10, 1815 +Bio: Ada was a British mathematician and writer, chiefly known for her work on + Charles Babbage's early mechanical general-purpose computer, the Analytical + Engine + +Name: Alan Turing +Birthday: June 23, 1912 +Bio: Alan was a British pioneering computer scientist, mathematician, logician, + cryptanalyst and theoretical biologist +``` + +## Installation + +``` +$ go get -v github.com/gosuri/uitable +``` + + +[![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/gosuri/uitable/trend.png)](https://bitdeli.com/free "Bitdeli Badge") + diff --git a/vendor/github.com/gosuri/uitable/table.go b/vendor/github.com/gosuri/uitable/table.go new file mode 100644 index 0000000000..b792c88367 --- /dev/null +++ b/vendor/github.com/gosuri/uitable/table.go @@ -0,0 +1,203 @@ +// Package uitable provides a decorator for formating data as a table +package uitable + +import ( + "fmt" + "strings" + "sync" + + "github.com/gosuri/uitable/util/strutil" + "github.com/gosuri/uitable/util/wordwrap" + "github.com/mattn/go-runewidth" +) + +// Separator is the default column seperator +var Separator = "\t" + +// Table represents a decorator that renders the data in formatted in a table +type Table struct { + // Rows is the collection of rows in the table + Rows []*Row + + // MaxColWidth is the maximum allowed width for cells in the table + MaxColWidth uint + + // Wrap when set to true wraps the contents of the columns when the length exceeds the MaxColWidth + Wrap bool + + // Separator is the seperator for columns in the table. Default is "\t" + Separator string + + mtx *sync.RWMutex + rightAlign map[int]bool +} + +// New returns a new Table with default values +func New() *Table { + return &Table{ + Separator: Separator, + mtx: new(sync.RWMutex), + rightAlign: map[int]bool{}, + } +} + +// AddRow adds a new row to the table +func (t *Table) AddRow(data ...interface{}) *Table { + t.mtx.Lock() + defer t.mtx.Unlock() + r := NewRow(data...) + t.Rows = append(t.Rows, r) + return t +} + +// Bytes returns the []byte value of table +func (t *Table) Bytes() []byte { + return []byte(t.String()) +} + +func (t *Table) RightAlign(col int) { + t.mtx.Lock() + t.rightAlign[col] = true + t.mtx.Unlock() +} + +// String returns the string value of table +func (t *Table) String() string { + t.mtx.RLock() + defer t.mtx.RUnlock() + + if len(t.Rows) == 0 { + return "" + } + + // determine the width for each column (cell in a row) + var colwidths []uint + for _, row := range t.Rows { + for i, cell := range row.Cells { + // resize colwidth array + if i+1 > len(colwidths) { + colwidths = append(colwidths, 0) + } + cellwidth := cell.LineWidth() + if t.MaxColWidth != 0 && cellwidth > t.MaxColWidth { + cellwidth = t.MaxColWidth + } + + if cellwidth > colwidths[i] { + colwidths[i] = cellwidth + } + } + } + + var lines []string + for _, row := range t.Rows { + row.Separator = t.Separator + for i, cell := range row.Cells { + cell.Width = colwidths[i] + cell.Wrap = t.Wrap + cell.RightAlign = t.rightAlign[i] + } + lines = append(lines, row.String()) + } + return strutil.Join(lines, "\n") +} + +// Row represents a row in a table +type Row struct { + // Cells is the group of cell for the row + Cells []*Cell + + // Separator for tabular columns + Separator string +} + +// NewRow returns a new Row and adds the data to the row +func NewRow(data ...interface{}) *Row { + r := &Row{Cells: make([]*Cell, len(data))} + for i, d := range data { + r.Cells[i] = &Cell{Data: d} + } + return r +} + +// String returns the string representation of the row +func (r *Row) String() string { + // get the max number of lines for each cell + var lc int // line count + for _, cell := range r.Cells { + if clc := len(strings.Split(cell.String(), "\n")); clc > lc { + lc = clc + } + } + + // allocate a two-dimentional array of cells for each line and add size them + cells := make([][]*Cell, lc) + for x := 0; x < lc; x++ { + cells[x] = make([]*Cell, len(r.Cells)) + for y := 0; y < len(r.Cells); y++ { + cells[x][y] = &Cell{Width: r.Cells[y].Width} + } + } + + // insert each line in a cell as new cell in the cells array + for y, cell := range r.Cells { + lines := strings.Split(cell.String(), "\n") + for x, line := range lines { + cells[x][y].Data = line + } + } + + // format each line + lines := make([]string, lc) + for x := range lines { + line := make([]string, len(cells[x])) + for y := range cells[x] { + line[y] = cells[x][y].String() + } + lines[x] = strutil.Join(line, r.Separator) + } + return strutil.Join(lines, "\n") +} + +// Cell represents a column in a row +type Cell struct { + // Width is the width of the cell + Width uint + + // Wrap when true wraps the contents of the cell when the lenght exceeds the width + Wrap bool + + // RightAlign when true aligns contents to the right + RightAlign bool + + // Data is the cell data + Data interface{} +} + +// LineWidth returns the max width of all the lines in a cell +func (c *Cell) LineWidth() uint { + width := 0 + for _, s := range strings.Split(c.String(), "\n") { + w := runewidth.StringWidth(s) + if w > width { + width = w + } + } + return uint(width) +} + +// String returns the string formated representation of the cell +func (c *Cell) String() string { + if c.Data == nil { + return strutil.PadLeft(" ", int(c.Width), ' ') + } + s := fmt.Sprintf("%v", c.Data) + if c.Width > 0 { + if c.Wrap && uint(len(s)) > c.Width { + return wordwrap.WrapString(s, c.Width) + } else { + return strutil.Resize(s, c.Width, c.RightAlign) + } + } + return s +} diff --git a/vendor/github.com/gosuri/uitable/util/strutil/strutil.go b/vendor/github.com/gosuri/uitable/util/strutil/strutil.go new file mode 100644 index 0000000000..cb35bce272 --- /dev/null +++ b/vendor/github.com/gosuri/uitable/util/strutil/strutil.go @@ -0,0 +1,76 @@ +// Package strutil provides various utilities for manipulating strings +package strutil + +import ( + "bytes" + "github.com/mattn/go-runewidth" +) + +// PadRight returns a new string of a specified length in which the end of the current string is padded with spaces or with a specified Unicode character. +func PadRight(str string, length int, pad byte) string { + slen := runewidth.StringWidth(str) + if slen >= length { + return str + } + buf := bytes.NewBufferString(str) + for i := 0; i < length-slen; i++ { + buf.WriteByte(pad) + } + return buf.String() +} + +// PadLeft returns a new string of a specified length in which the beginning of the current string is padded with spaces or with a specified Unicode character. +func PadLeft(str string, length int, pad byte) string { + slen := runewidth.StringWidth(str) + if slen >= length { + return str + } + var buf bytes.Buffer + for i := 0; i < length-slen; i++ { + buf.WriteByte(pad) + } + buf.WriteString(str) + return buf.String() +} + +// Resize resizes the string with the given length. It ellipses with '...' when the string's length exceeds +// the desired length or pads spaces to the right of the string when length is smaller than desired +func Resize(s string, length uint, rightAlign bool) string { + slen := runewidth.StringWidth(s) + n := int(length) + if slen == n { + return s + } + // Pads only when length of the string smaller than len needed + if rightAlign { + s = PadLeft(s, n, ' ') + } else { + s = PadRight(s, n, ' ') + } + if slen > n { + rs := []rune(s) + var buf bytes.Buffer + w := 0 + for _, r := range rs { + buf.WriteRune(r) + rw := runewidth.RuneWidth(r) + if w+rw >= n-3 { + break + } + w += rw + } + buf.WriteString("...") + s = buf.String() + } + return s +} + +// Join joins the list of the string with the delim provided +func Join(list []string, delim string) string { + var buf bytes.Buffer + for i := 0; i < len(list)-1; i++ { + buf.WriteString(list[i] + delim) + } + buf.WriteString(list[len(list)-1]) + return buf.String() +} diff --git a/vendor/github.com/gosuri/uitable/util/wordwrap/LICENSE.md b/vendor/github.com/gosuri/uitable/util/wordwrap/LICENSE.md new file mode 100644 index 0000000000..2298515904 --- /dev/null +++ b/vendor/github.com/gosuri/uitable/util/wordwrap/LICENSE.md @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Mitchell Hashimoto + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/gosuri/uitable/util/wordwrap/README.md b/vendor/github.com/gosuri/uitable/util/wordwrap/README.md new file mode 100644 index 0000000000..60ae311700 --- /dev/null +++ b/vendor/github.com/gosuri/uitable/util/wordwrap/README.md @@ -0,0 +1,39 @@ +# go-wordwrap + +`go-wordwrap` (Golang package: `wordwrap`) is a package for Go that +automatically wraps words into multiple lines. The primary use case for this +is in formatting CLI output, but of course word wrapping is a generally useful +thing to do. + +## Installation and Usage + +Install using `go get github.com/mitchellh/go-wordwrap`. + +Full documentation is available at +http://godoc.org/github.com/mitchellh/go-wordwrap + +Below is an example of its usage ignoring errors: + +```go +wrapped := wordwrap.WrapString("foo bar baz", 3) +fmt.Println(wrapped) +``` + +Would output: + +``` +foo +bar +baz +``` + +## Word Wrap Algorithm + +This library doesn't use any clever algorithm for word wrapping. The wrapping +is actually very naive: whenever there is whitespace or an explicit linebreak. +The goal of this library is for word wrapping CLI output, so the input is +typically pretty well controlled human language. Because of this, the naive +approach typically works just fine. + +In the future, we'd like to make the algorithm more advanced. We would do +so without breaking the API. diff --git a/vendor/github.com/gosuri/uitable/util/wordwrap/wordwrap.go b/vendor/github.com/gosuri/uitable/util/wordwrap/wordwrap.go new file mode 100644 index 0000000000..e64dbc8016 --- /dev/null +++ b/vendor/github.com/gosuri/uitable/util/wordwrap/wordwrap.go @@ -0,0 +1,84 @@ +// Package wordwrap provides methods for wrapping the contents of a string +package wordwrap + +import ( + "bytes" + "github.com/mattn/go-runewidth" + "unicode" +) + +// WrapString wraps the given string within lim width in characters. +// +// Wrapping is currently naive and only happens at white-space. A future +// version of the library will implement smarter wrapping. This means that +// pathological cases can dramatically reach past the limit, such as a very +// long word. +func WrapString(s string, lim uint) string { + // Initialize a buffer with a slightly larger size to account for breaks + init := make([]byte, 0, len(s)) + buf := bytes.NewBuffer(init) + + var current uint + var wordBuf, spaceBuf bytes.Buffer + var wordWidth, spaceWidth int + + for _, char := range s { + if char == '\n' { + if wordBuf.Len() == 0 { + if current+uint(spaceWidth) > lim { + current = 0 + } else { + current += uint(spaceWidth) + spaceBuf.WriteTo(buf) + spaceWidth += runewidth.StringWidth(buf.String()) + } + spaceBuf.Reset() + spaceWidth = 0 + } else { + current += uint(spaceWidth + wordWidth) + spaceBuf.WriteTo(buf) + spaceBuf.Reset() + wordBuf.WriteTo(buf) + wordBuf.Reset() + spaceWidth = 0 + wordWidth = 0 + } + buf.WriteRune(char) + current = 0 + } else if unicode.IsSpace(char) { + if spaceBuf.Len() == 0 || wordBuf.Len() > 0 { + current += uint(spaceWidth + wordWidth) + spaceBuf.WriteTo(buf) + spaceBuf.Reset() + wordBuf.WriteTo(buf) + wordBuf.Reset() + spaceWidth = 0 + wordWidth = 0 + } + + spaceBuf.WriteRune(char) + spaceWidth += runewidth.RuneWidth(char) + } else { + wordBuf.WriteRune(char) + wordWidth += runewidth.RuneWidth(char) + + if current+uint(spaceWidth+wordWidth) > lim && uint(wordWidth) < lim { + buf.WriteRune('\n') + current = 0 + spaceBuf.Reset() + spaceWidth = 0 + } + } + } + + if wordBuf.Len() == 0 { + if current+uint(spaceWidth) <= lim { + spaceBuf.WriteTo(buf) + } + } else { + spaceBuf.WriteTo(buf) + wordBuf.WriteTo(buf) + } + + return buf.String() +} diff --git a/vendor/github.com/vmware/govmomi/simulator/license_manager.go b/vendor/github.com/vmware/govmomi/simulator/license_manager.go deleted file mode 100644 index 13565d32ef..0000000000 --- a/vendor/github.com/vmware/govmomi/simulator/license_manager.go +++ /dev/null @@ -1,156 +0,0 @@ -/* -Copyright (c) 2017 VMware, Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -// Copyright 2017 VMware, Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package simulator - -import ( - "github.com/vmware/govmomi/object" - "github.com/vmware/govmomi/vim25/methods" - "github.com/vmware/govmomi/vim25/mo" - "github.com/vmware/govmomi/vim25/soap" - "github.com/vmware/govmomi/vim25/types" -) - -// EvalLicense is the default license -var EvalLicense = types.LicenseManagerLicenseInfo{ - LicenseKey: "00000-00000-00000-00000-00000", - EditionKey: "eval", - Name: "Evaluation Mode", - Properties: []types.KeyAnyValue{ - { - Key: "feature", - Value: types.KeyValue{ - Key: "serialuri:2", - Value: "Remote virtual Serial Port Concentrator", - }, - }, - { - Key: "feature", - Value: types.KeyValue{ - Key: "dvs", - Value: "vSphere Distributed Switch", - }, - }, - }, -} - -type LicenseManager struct { - mo.LicenseManager -} - -func NewLicenseManager(ref types.ManagedObjectReference) object.Reference { - m := &LicenseManager{} - m.Self = ref - m.Licenses = []types.LicenseManagerLicenseInfo{EvalLicense} - - if Map.IsVPX() { - am := Map.Put(&LicenseAssignmentManager{}).Reference() - m.LicenseAssignmentManager = &am - } - - return m -} - -func (m *LicenseManager) AddLicense(req *types.AddLicense) soap.HasFault { - body := &methods.AddLicenseBody{ - Res: &types.AddLicenseResponse{}, - } - - for _, license := range m.Licenses { - if license.LicenseKey == req.LicenseKey { - body.Res.Returnval = licenseInfo(license.LicenseKey, license.Labels) - return body - } - } - - m.Licenses = append(m.Licenses, types.LicenseManagerLicenseInfo{ - LicenseKey: req.LicenseKey, - Labels: req.Labels, - }) - - body.Res.Returnval = licenseInfo(req.LicenseKey, req.Labels) - - return body -} - -func (m *LicenseManager) RemoveLicense(req *types.RemoveLicense) soap.HasFault { - body := &methods.RemoveLicenseBody{ - Res: &types.RemoveLicenseResponse{}, - } - - for i, license := range m.Licenses { - if req.LicenseKey == license.LicenseKey { - m.Licenses = append(m.Licenses[:i], m.Licenses[i+1:]...) - return body - } - } - return body -} - -type LicenseAssignmentManager struct { - mo.LicenseAssignmentManager -} - -func (m *LicenseAssignmentManager) QueryAssignedLicenses(req *types.QueryAssignedLicenses) soap.HasFault { - body := &methods.QueryAssignedLicensesBody{ - Res: &types.QueryAssignedLicensesResponse{}, - } - - // EntityId can be a HostSystem or the vCenter InstanceUuid - if req.EntityId != "" { - if req.EntityId != Map.content().About.InstanceUuid { - id := types.ManagedObjectReference{ - Type: "HostSystem", - Value: req.EntityId, - } - - if Map.Get(id) == nil { - return body - } - } - } - - body.Res.Returnval = []types.LicenseAssignmentManagerLicenseAssignment{ - { - EntityId: req.EntityId, - AssignedLicense: EvalLicense, - }, - } - - return body -} - -func licenseInfo(key string, labels []types.KeyValue) types.LicenseManagerLicenseInfo { - info := EvalLicense - - info.LicenseKey = key - info.Labels = labels - - return info -} diff --git a/vendor/yunion.io/x/jsonutils/marshal.go b/vendor/yunion.io/x/jsonutils/marshal.go index bedce745e9..aff79cf1d2 100644 --- a/vendor/yunion.io/x/jsonutils/marshal.go +++ b/vendor/yunion.io/x/jsonutils/marshal.go @@ -12,8 +12,8 @@ import ( "reflect" "time" - "yunion.io/x/pkg/gotypes" "yunion.io/x/log" + "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/tristate" "yunion.io/x/pkg/util/reflectutils" "yunion.io/x/pkg/util/timeutils" @@ -58,6 +58,9 @@ func struct2JSONPairs(val reflect.Value) []JSONPair { objPairs = append(objPairs, newPairs...) } else { key := reflectutils.GetStructFieldName(&fieldType) // utils.CamelSplit(fieldType.Name, "_") + if key == "" { + continue + } val := marshalValue(val.Field(i)) if val != JSONNull { objPair := JSONPair{key: key, val: val} diff --git a/vendor/yunion.io/x/jsonutils/unmarshal.go b/vendor/yunion.io/x/jsonutils/unmarshal.go index 3a98a40490..941fcdf729 100644 --- a/vendor/yunion.io/x/jsonutils/unmarshal.go +++ b/vendor/yunion.io/x/jsonutils/unmarshal.go @@ -47,8 +47,6 @@ func jsonUnmarshal(jo JSONObject, o interface{}, keys []string) error { } func (this *JSONValue) unmarshalValue(val reflect.Value) error { - // return fmt.Errorf("JSONValue: type mismatch") - // null value if val.CanSet() { zeroVal := reflect.New(val.Type()).Elem() val.Set(zeroVal) @@ -102,6 +100,11 @@ func (this *JSONInt) unmarshalValue(val reflect.Value) error { } case reflect.String: val.SetString(fmt.Sprintf("%d", this.data)) + case reflect.Ptr: + if val.IsNil() { + val.Set(reflect.New(val.Type().Elem())) + } + return this.unmarshalValue(val.Elem()) default: return fmt.Errorf("JSONInt type mismatch: %s", val.Type()) } @@ -162,6 +165,11 @@ func (this *JSONBool) unmarshalValue(val reflect.Value) error { } else { val.SetString("false") } + case reflect.Ptr: + if val.IsNil() { + val.Set(reflect.New(val.Type().Elem())) + } + return this.unmarshalValue(val.Elem()) default: return fmt.Errorf("JSONBool type mismatch: %s", val.Type()) } @@ -231,6 +239,11 @@ func (this *JSONFloat) unmarshalValue(val reflect.Value) error { } case reflect.String: val.SetString(fmt.Sprintf("%f", this.data)) + case reflect.Ptr: + if val.IsNil() { + val.Set(reflect.New(val.Type().Elem())) + } + return this.unmarshalValue(val.Elem()) default: return fmt.Errorf("JSONFloat type mismatch: %s", val.Type()) } @@ -314,6 +327,11 @@ func (this *JSONString) unmarshalValue(val reflect.Value) error { val.SetBool(utils.ToBool(this.data)) case reflect.String: val.SetString(this.data) + case reflect.Ptr: + if val.IsNil() { + val.Set(reflect.New(val.Type().Elem())) + } + return this.unmarshalValue(val.Elem()) default: return fmt.Errorf("JSONString type mismatch: %s", val.Type()) } @@ -339,15 +357,35 @@ func (this *JSONArray) unmarshalValue(val reflect.Value) error { case reflect.String: val.SetString(this.String()) return nil + case reflect.Ptr: + kind := val.Type().Elem().Kind() + if kind == reflect.Array || kind == reflect.Slice { + if val.IsNil() { + val.Set(reflect.New(val.Type().Elem())) + } + return this.unmarshalValue(val.Elem()) + } + return fmt.Errorf("JSONArray type mismatch %s", val.Type()) case reflect.Slice, reflect.Array: - for _, json := range this.data { - newEle := reflect.New(val.Type().Elem()).Elem() - err := json.unmarshalValue(newEle) + if val.Kind() == reflect.Array { + if val.Len() != len(this.data) { + return fmt.Errorf("JSONArray length unmatch %s: %d != %d", + val.Type(), val.Len(), len(this.data)) + } + } else if val.Kind() == reflect.Slice { + dataLen := len(this.data) + if val.Cap() < dataLen { + newVal := reflect.MakeSlice(val.Type(), dataLen, dataLen) + val.Set(newVal) + } else if val.Len() != dataLen { + val.SetLen(dataLen) + } + } + for i, json := range this.data { + err := json.unmarshalValue(val.Index(i)) if err != nil { return err } - newVal := reflect.Append(val, newEle) - val.Set(newVal) } default: return fmt.Errorf("JSONArray type mismatch: %s", val.Type()) @@ -376,6 +414,16 @@ func (this *JSONDict) unmarshalValue(val reflect.Value) error { return this.unmarshalMap(val) case reflect.Struct: return this.unmarshalStruct(val) + case reflect.Ptr: + kind := val.Type().Elem().Kind() + if kind == reflect.Struct || kind == reflect.Map { + if val.IsNil() { + newVal := reflect.New(val.Type().Elem()) + val.Set(newVal) + } + return this.unmarshalValue(val.Elem()) + } + fallthrough default: return fmt.Errorf("JSONDict type mismatch: %s", val.Type()) } diff --git a/vendor/yunion.io/x/log/hooks/file.go b/vendor/yunion.io/x/log/hooks/file.go new file mode 100644 index 0000000000..3a99b0397a --- /dev/null +++ b/vendor/yunion.io/x/log/hooks/file.go @@ -0,0 +1,108 @@ +package hooks + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/sirupsen/logrus" +) + +type LogFileHook struct { + FileDir string + FileName string + fullPath string + file *os.File + written int64 +} + +func (h *LogFileHook) Init() error { + if fi, err := os.Lstat(h.FileDir); err != nil { + if os.IsNotExist(err) { + os.MkdirAll(h.FileDir, 0755) + } else { + return fmt.Errorf("Lstat %s: %s", h.FileDir, err) + } + } else if !fi.Mode().IsDir() { + return fmt.Errorf("%s exists and it's not a directory", h.FileDir) + } + + h.fullPath = filepath.Join(h.FileDir, h.FileName) + file, err := os.OpenFile(h.fullPath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) + if err != nil { + return fmt.Errorf("OpenFile %s: %s", h.fullPath, err) + } + h.file = file + h.written = 0 + return nil +} + +func (h *LogFileHook) DeInit() { + if h.file != nil { + h.file.Close() + } +} + +func (h *LogFileHook) Levels() []logrus.Level { + return logrus.AllLevels +} + +func (h *LogFileHook) Fire(e *logrus.Entry) error { + if b, err := e.Logger.Formatter.Format(e); err != nil { + return err + } else { + n, err := h.file.Write(b) + h.written += int64(n) + return err + } + return nil +} + +func (h *LogFileHook) Written() int64 { + return h.written +} + +// rotate by size +type LogFileRotateHook struct { + LogFileHook + RotateNum int + RotateSize int64 + filePaths []string +} + +func (h *LogFileRotateHook) Init() error { + if err := h.LogFileHook.Init(); err != nil { + return err + } + h.filePaths = make([]string, h.RotateNum) + for i := 1; i < h.RotateNum; i++ { + fileName := fmt.Sprintf("%s.%d", h.FileName, i) + filePath := filepath.Join(h.FileDir, fileName) + h.filePaths[i] = filePath + } + h.filePaths[0] = filepath.Join(h.FileDir, h.FileName) + return nil +} + +func (h *LogFileRotateHook) rotate() { + for i := h.RotateNum - 1; i > 0; i-- { + filePath0 := h.filePaths[i-1] + if _, err := os.Lstat(filePath0); err != nil { + continue + } + filePath1 := h.filePaths[i] + os.Rename(filePath0, filePath1) + } + h.LogFileHook.DeInit() + h.LogFileHook.Init() +} + +func (h *LogFileRotateHook) Fire(e *logrus.Entry) error { + if err := h.LogFileHook.Fire(e); err != nil { + return err + } + if h.LogFileHook.Written() >= h.RotateSize { + h.rotate() + } + return nil +} diff --git a/vendor/yunion.io/x/log/hooks/stdio.go b/vendor/yunion.io/x/log/hooks/stdio.go index 6fa3dc4381..e5d32000ad 100644 --- a/vendor/yunion.io/x/log/hooks/stdio.go +++ b/vendor/yunion.io/x/log/hooks/stdio.go @@ -12,14 +12,9 @@ type StdioHook struct{} func (hook *StdioHook) Fire(entry *logrus.Entry) error { line, err := entry.String() if err != nil { - fmt.Fprintf(os.Stderr, "Unable to read entry, %v", err) return err } - if entry.Level >= logrus.ErrorLevel { - fmt.Fprintf(os.Stderr, line) - } else { - fmt.Fprintf(os.Stdout, line) - } + fmt.Fprintf(os.Stderr, line) return nil } diff --git a/vendor/yunion.io/x/pkg/gotypes/deepcopy.go b/vendor/yunion.io/x/pkg/gotypes/deepcopy.go new file mode 100644 index 0000000000..f8a186c6d1 --- /dev/null +++ b/vendor/yunion.io/x/pkg/gotypes/deepcopy.go @@ -0,0 +1,100 @@ +package gotypes + +import ( + "reflect" +) + +type DeepCopyFlags uintptr + +func DeepCopy(v interface{}) interface{} { + rv := reflect.ValueOf(v) + cpRv := DeepCopyRv(rv) + cpV := cpRv.Interface() + return cpV +} + +func DeepCopyRv(rv reflect.Value) reflect.Value { + kind := rv.Kind() + if kind == reflect.Invalid { + return reflect.Value{} + } + typ := rv.Type() + switch kind { + case reflect.Ptr: + if rv.IsNil() { + return reflect.New(typ).Elem() + } + elemRv := rv.Elem() + cpElemRv := DeepCopyRv(elemRv) + cpRv := reflect.New(typ.Elem()) + cpRv.Elem().Set(cpElemRv) + return cpRv + case reflect.Slice, reflect.Array: + var cpRv reflect.Value + switch kind { + case reflect.Slice: + if rv.IsNil() { + return reflect.New(typ).Elem() + } + cpRv = reflect.MakeSlice(typ, rv.Len(), rv.Cap()) + case reflect.Array: + cpRv = reflect.New(typ).Elem() + } + n := rv.Len() + for i := 0; i < n; i++ { + elem := rv.Index(i) + cpElem := DeepCopyRv(elem) + cpRvElem := cpRv.Index(i) + cpRvElem.Set(cpElem) + } + return cpRv + case reflect.Struct: + cpRv := reflect.New(typ).Elem() + if typ == TimeType { + cpRv.Set(rv) + } + n := rv.NumField() + for i := 0; i < n; i++ { + f := rv.Field(i) + if f.CanInterface() { + cpRvF := cpRv.Field(i) + cpF := DeepCopyRv(f) + cpRvF.Set(cpF) + } + } + return cpRv + case reflect.Map: + if rv.IsNil() { + return reflect.New(typ).Elem() + } + cpRv := reflect.MakeMap(typ) + mk := rv.MapKeys() + for _, k := range mk { + v := rv.MapIndex(k) + cpK := DeepCopyRv(k) + cpV := DeepCopyRv(v) + cpRv.SetMapIndex(cpK, cpV) + } + return cpRv + case reflect.Interface: + cpRv := reflect.New(typ).Elem() + if !rv.IsNil() { + elemRv := rv.Elem() + cpElemRv := DeepCopyRv(elemRv) + cpRv.Set(cpElemRv) + } + return cpRv + case reflect.Chan: + if rv.IsNil() { + return reflect.New(typ).Elem() + } + return reflect.MakeChan(typ.Elem(), rv.Cap()) + default: + // Invalid, Bool, Int*, Uint*, Float*, Complex*, Func, String + // TODO UnsafePointer + cpRv := reflect.New(typ).Elem() + cpRv.Set(rv) + return cpRv + } + panic("unhandled kind " + kind.String()) +} diff --git a/vendor/yunion.io/x/pkg/gotypes/serialize.go b/vendor/yunion.io/x/pkg/gotypes/serialize.go index 8cb8c41d07..fdbc271497 100644 --- a/vendor/yunion.io/x/pkg/gotypes/serialize.go +++ b/vendor/yunion.io/x/pkg/gotypes/serialize.go @@ -14,30 +14,32 @@ type ISerializable interface { type FuncSerializableAllocator func() ISerializable -var ISerializableType reflect.Type -var serializableAllocators map[reflect.Type]FuncSerializableAllocator -var ErrTypeNotSerializable error - -func init() { - ISerializableType = reflect.TypeOf((*ISerializable)(nil)).Elem() - serializableAllocators = make(map[reflect.Type]FuncSerializableAllocator) +var ( + ISerializableType = reflect.TypeOf((*ISerializable)(nil)).Elem() + serializableAllocators = map[reflect.Type]FuncSerializableAllocator{} ErrTypeNotSerializable = errors.New("Type not serializable") -} +) +// RegisterSerializable registers an allocator func for the specified serializable type. +// +// This is intended to be used when you have multiple implmenetations of an +// interface and you want to use only one of them to cover them all. +// TokenCredential and SSimpleToken is such a case. func RegisterSerializable(valType reflect.Type, alloc FuncSerializableAllocator) { + if !IsSerializable(valType) { + panic(valType.String() + " does not implement ISerializable") + } serializableAllocators[valType] = alloc } -func IsSerializable(objType reflect.Type) bool { - _, ok := serializableAllocators[objType] - return ok +func IsSerializable(valType reflect.Type) bool { + return valType.Implements(ISerializableType) } func NewSerializable(objType reflect.Type) (ISerializable, error) { deserFunc, ok := serializableAllocators[objType] - if !ok { - return nil, ErrTypeNotSerializable - } else { + if ok { return deserFunc(), nil } + return nil, ErrTypeNotSerializable } diff --git a/vendor/yunion.io/x/pkg/prettytable/prettytable.go b/vendor/yunion.io/x/pkg/prettytable/prettytable.go index 8de58e04b0..c0e10aa550 100644 --- a/vendor/yunion.io/x/pkg/prettytable/prettytable.go +++ b/vendor/yunion.io/x/pkg/prettytable/prettytable.go @@ -2,6 +2,7 @@ package prettytable import ( "bytes" + "strings" ) type AlignmentType uint8 @@ -40,6 +41,7 @@ func rowLine(buf *bytes.Buffer, widths []int) { buf.WriteByte('-') buf.WriteByte('+') } + buf.WriteByte('\n') } func textCell(buf *bytes.Buffer, col ptColumn, width int) { @@ -65,13 +67,45 @@ func textCell(buf *bytes.Buffer, col ptColumn, width int) { } func textLine(buf *bytes.Buffer, columns []ptColumn, widths []int) { - buf.WriteByte('|') - for i, w := range widths { - buf.WriteByte(' ') - textCell(buf, columns[i], w) - buf.WriteByte(' ') - buf.WriteByte('|') + nlines := 0 + splitted := make([][]string, len(columns)) + for i, column := range columns { + title := strings.TrimRight(column.Title, "\n") + s := strings.Split(title, "\n") + splitted[i] = s + if nlines < len(s) { + nlines = len(s) + } } + for j := 0; j < nlines; j++ { + buf.WriteByte('|') + for i, w := range widths { + buf.WriteByte(' ') + var text string + if j < len(splitted[i]) { + text = splitted[i][j] + } + c := ptColumn{ + Align: columns[i].Align, + Title: text, + } + textCell(buf, c, w) + buf.WriteByte(' ') + buf.WriteByte('|') + } + buf.WriteByte('\n') + } +} + +func cellWidth(cell string) int { + width := 0 + lines := strings.Split(cell, "\n") + for _, line := range lines { + if width < len(line) { + width = len(line) + } + } + return width } func (this *PrettyTable) GetString(fields [][]string) string { @@ -80,37 +114,32 @@ func (this *PrettyTable) GetString(fields [][]string) string { } var widths = make([]int, len(this.columns)) for i, c := range this.columns { - if widths[i] < len(c.Title) { - widths[i] = len(c.Title) - } + widths[i] = cellWidth(c.Title) } for _, line := range fields { for i, cell := range line { - if widths[i] < len(cell) { - widths[i] = len(cell) + cw := cellWidth(cell) + if widths[i] < cw { + widths[i] = cw } } } var columns = make([]ptColumn, len(this.columns)) var buf bytes.Buffer rowLine(&buf, widths) - buf.WriteByte('\n') for i := 0; i < len(columns); i++ { columns[i].Title = this.columns[i].Title columns[i].Align = AlignCenter } textLine(&buf, columns, widths) - buf.WriteByte('\n') rowLine(&buf, widths) for _, line := range fields { for i := 0; i < len(line); i++ { columns[i].Title = line[i] columns[i].Align = this.columns[i].Align } - buf.WriteByte('\n') textLine(&buf, columns, widths) } - buf.WriteByte('\n') rowLine(&buf, widths) return buf.String() } diff --git a/vendor/yunion.io/x/pkg/util/filterclause/filterclause.go b/vendor/yunion.io/x/pkg/util/filterclause/filterclause.go index 2a3a1ccd10..5518ea0f3b 100644 --- a/vendor/yunion.io/x/pkg/util/filterclause/filterclause.go +++ b/vendor/yunion.io/x/pkg/util/filterclause/filterclause.go @@ -16,6 +16,20 @@ type SFilterClause struct { params []string } +type SJointFilterClause struct { + SFilterClause + JointModel string + ReleatedKey string +} + +func (jfc *SJointFilterClause) GetJointFilter(q *sqlchemy.SQuery) sqlchemy.ICondition { + return jfc.QueryCondition(q) +} + +func (jfc *SJointFilterClause) GetJointModelName() string { + return jfc.JointModel[:len(jfc.JointModel)-1] +} + func (fc *SFilterClause) QueryCondition(q *sqlchemy.SQuery) sqlchemy.ICondition { field := q.Field(fc.field) if field == nil { @@ -63,11 +77,13 @@ func (fc *SFilterClause) String() string { } var ( - filterClausePattern *regexp.Regexp + filterClausePattern *regexp.Regexp + jointFilterClausePattern *regexp.Regexp ) func init() { filterClausePattern = regexp.MustCompile(`^(\w+)\.(\w+)\((.*)\)`) + jointFilterClausePattern = regexp.MustCompile(`^(\w+)\((\w+)\).(\w+)\.(\w+)\((.*)\)`) } func ParseFilterClause(filter string) *SFilterClause { @@ -79,3 +95,21 @@ func ParseFilterClause(filter string) *SFilterClause { fc := SFilterClause{field: matches[1], funcName: matches[2], params: params} return &fc } + +func ParseJointFilterClause(jointFilter string) *SJointFilterClause { + matches := jointFilterClausePattern.FindStringSubmatch(jointFilter) + if matches == nil { + return nil + } + params := utils.FindWords([]byte(matches[5]), 0) + jfc := SJointFilterClause{ + SFilterClause: SFilterClause{ + field: matches[3], + funcName: matches[4], + params: params, + }, + JointModel: matches[1], + ReleatedKey: matches[2], + } + return &jfc +} diff --git a/vendor/yunion.io/x/pkg/util/reflectutils/reflectutils.go b/vendor/yunion.io/x/pkg/util/reflectutils/reflectutils.go index 6b58992180..55d9019d8e 100644 --- a/vendor/yunion.io/x/pkg/util/reflectutils/reflectutils.go +++ b/vendor/yunion.io/x/pkg/util/reflectutils/reflectutils.go @@ -2,6 +2,7 @@ package reflectutils import ( "reflect" + "strings" "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/utils" @@ -15,14 +16,23 @@ func GetStructFieldName(field *reflect.StructField) string { return nameStr } else { jsonStr, _ := tagMap["json"] - if len(jsonStr) > 0 { - return jsonStr - } else { - return utils.CamelSplit(field.Name, "_") - } + return toJsonKey(field.Name, jsonStr) } } +func toJsonKey(fieldName, jsonTag string) string { + jsonTag = strings.Replace(jsonTag, "omitempty", "", -1) + words := utils.FindWords([]byte(jsonTag), 0) + if len(words) == 0 { + return utils.CamelSplit(fieldName, "_") + } + name := words[0] + if name == "-" { + return "" + } + return name +} + func FetchStructFieldNameValueInterfaces(dataValue reflect.Value) map[string]interface{} { fields := make(map[string]interface{}) fetchStructFieldNameValueInterfaces(dataValue.Type(), dataValue, fields) diff --git a/vendor/yunion.io/x/pkg/util/sysutils/storagetypes.go b/vendor/yunion.io/x/pkg/util/sysutils/storagetypes.go index 476aa53bff..16baa403d2 100644 --- a/vendor/yunion.io/x/pkg/util/sysutils/storagetypes.go +++ b/vendor/yunion.io/x/pkg/util/sysutils/storagetypes.go @@ -9,12 +9,18 @@ const ( STORAGE_NAS = "nas" STORAGE_VSAN = "vsan" + STORAGE_CLOUD = "cloud" STORAGE_CLOUD_SSD = "cloud_ssd" STORAGE_CLOUD_EFFICIENCY = "cloud_efficiency" + + STORAGE_STANDARD = "standard" //Azure hdd storage type + STORAGE_PREMIUM = "premium" //Azure ssd storage type ) var STORAGE_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_SHEEPDOG, - STORAGE_RBD, STORAGE_DOCKER, STORAGE_NAS, STORAGE_VSAN, STORAGE_CLOUD_SSD, STORAGE_CLOUD_EFFICIENCY} + STORAGE_RBD, STORAGE_DOCKER, STORAGE_NAS, STORAGE_VSAN, + STORAGE_CLOUD, STORAGE_CLOUD_SSD, STORAGE_CLOUD_EFFICIENCY, + STORAGE_STANDARD, STORAGE_PREMIUM} var LOCAL_STORAGE_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL} diff --git a/vendor/yunion.io/x/pkg/utils/utils.go b/vendor/yunion.io/x/pkg/utils/utils.go index 4f61b411f6..0564c4b7fc 100644 --- a/vendor/yunion.io/x/pkg/utils/utils.go +++ b/vendor/yunion.io/x/pkg/utils/utils.go @@ -149,3 +149,71 @@ func FloatRound(num float64, precision int) float64 { num = temp return num } + +func getStringInQuote(s string, start, length int, quote byte, backIndex int) (string, int) { + var i = start + 1 + if quote != '\'' && quote != '"' { + for i < length { + if s[i] == ' ' || s[i] == '\'' || s[i] == '"' { + return s[backIndex:i], i + } + i++ + } + return s[backIndex:], i + } else { + for i < length { + if s[i] == quote { + return s[backIndex+1 : i], i + 1 + } + i++ + } + return s[backIndex:], i + } +} + +func ArgsStringToArray(s string) []string { + var args, i, j = make([]string, 0), 0, 0 + var length = len(s) + for i < length { + switch s[i] { + case ' ': + if i > j { + args = append(args, s[j:i]) + j = i + } + i++ + j = i + case '"': + var s1, s2 string + var oldStr = s[j:i] + s1, i = getStringInQuote(s, i, length, '"', i) + s1 = oldStr + s1 + for i < length && s[i] != ' ' { + s2, i = getStringInQuote(s, i, length, s[i], i) + s1 += s2 + fmt.Println(s2) + } + args = append(args, s1) + i++ + j = i + case '\'': + var s1, s2 string + var oldStr = s[j:i] + s1, i = getStringInQuote(s, i, length, '\'', i) + s1 = oldStr + s1 + for i < length && s[i] != ' ' { + s2, i = getStringInQuote(s, i, length, s[i], i) + s1 += s2 + } + args = append(args, s1) + i++ + j = i + default: + i++ + } + } + if j < length { + args = append(args, s[j:]) + } + return args +} diff --git a/vendor/yunion.io/x/sqlchemy/column.go b/vendor/yunion.io/x/sqlchemy/column.go index 90ff609a11..8f65bdaf74 100644 --- a/vendor/yunion.io/x/sqlchemy/column.go +++ b/vendor/yunion.io/x/sqlchemy/column.go @@ -7,7 +7,6 @@ import ( "strings" "time" - "yunion.io/x/log" "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/tristate" "yunion.io/x/pkg/util/regutils" @@ -465,11 +464,11 @@ func (c *SDecimalColumn) IsZero(val interface{}) bool { func NewDecimalColumn(name string, tagmap map[string]string) SDecimalColumn { tagmap, v, ok := utils.TagPop(tagmap, TAG_PRECISION) if !ok { - log.Fatalf("Field %s of float should have precision tag", name) + panic(fmt.Sprintf("Field %q of float misses precision tag", name)) } prec, err := strconv.Atoi(v) if err != nil { - log.Fatalf("Field %s of float precision %s shoud be integer!", name, v) + panic(fmt.Sprintf("Field precision of %q shoud be integer (%q)", name, v)) } return SDecimalColumn{SBaseWidthColumn: NewBaseWidthColumn(name, "DECIMAL", tagmap), Precision: prec} @@ -544,7 +543,7 @@ func NewTextColumn(name string, tagmap map[string]string) STextColumn { if len(charset) == 0 { charset = "utf8" } else if charset != "utf8" && charset != "ascii" { - log.Fatalf("Unsupported charset %s for %s", charset, name) + panic(fmt.Sprintf("Unsupported charset %s for %s", charset, name)) } return STextColumn{SBaseWidthColumn: NewBaseWidthColumn(name, sqltype, tagmap), Charset: charset} diff --git a/vendor/yunion.io/x/sqlchemy/field_update.go b/vendor/yunion.io/x/sqlchemy/field_update.go index 2087dbb6c0..b6b32203a5 100644 --- a/vendor/yunion.io/x/sqlchemy/field_update.go +++ b/vendor/yunion.io/x/sqlchemy/field_update.go @@ -102,7 +102,7 @@ func (ts *STableSpec) updateFields(dt interface{}, fields map[string]interface{} buf.WriteString(fmt.Sprintf(", `%s` = `%s` + 1", versionField, versionField)) } for _, updatedField := range updatedFields { - buf.WriteString(fmt.Sprintf(", `%s` = NOW()", updatedField)) + buf.WriteString(fmt.Sprintf(", `%s` = UTC_TIMESTAMP()", updatedField)) } buf.WriteString(" WHERE ") first = true diff --git a/vendor/yunion.io/x/sqlchemy/insert.go b/vendor/yunion.io/x/sqlchemy/insert.go index 341ce91343..c7442588f0 100644 --- a/vendor/yunion.io/x/sqlchemy/insert.go +++ b/vendor/yunion.io/x/sqlchemy/insert.go @@ -13,15 +13,7 @@ func (t *STableSpec) Insert(dt interface{}) error { return t.insert(dt, false) } -func (t *STableSpec) insert(dt interface{}, debug bool) error { - beforeInsertFunc := reflect.ValueOf(dt).MethodByName("BeforeInsert") - if beforeInsertFunc.IsValid() && !beforeInsertFunc.IsNil() { - beforeInsertFunc.Call([]reflect.Value{}) - } - - // dataType := reflect.TypeOf(dt).Elem() - dataValue := reflect.ValueOf(dt).Elem() - +func (t *STableSpec) insertSqlPrep(dataFields map[string]interface{}) (string, []interface{}, error) { var autoIncField string createdAtFields := make([]string, 0) @@ -29,7 +21,6 @@ func (t *STableSpec) insert(dt interface{}, debug bool) error { format := make([]string, 0) values := make([]interface{}, 0) - fields := reflectutils.FetchStructFieldNameValueInterfaces(dataValue) for _, c := range t.columns { isAutoInc := false nc, ok := c.(*SIntegerColumn) @@ -40,13 +31,12 @@ func (t *STableSpec) insert(dt interface{}, debug bool) error { k := c.Name() dtc, ok := c.(*SDateTimeColumn) - ov := fields[k] + ov := dataFields[k] - // log.Debugf("field %s value %s %s", k, ov, ov==nil) if ok && (dtc.IsCreatedAt || dtc.IsUpdatedAt) { createdAtFields = append(createdAtFields, k) names = append(names, fmt.Sprintf("`%s`", k)) - format = append(format, "NOW()") + format = append(format, "UTC_TIMESTAMP()") } else if ov != nil && !c.IsZero(ov) && !isAutoInc { v := c.ConvertFromValue(ov) values = append(values, v) @@ -54,15 +44,14 @@ func (t *STableSpec) insert(dt interface{}, debug bool) error { format = append(format, "?") } else if c.IsPrimary() { if isAutoInc { - if len(autoIncField) == 0 { - autoIncField = k - } else { - log.Fatalf("multiple auto_increment columns???") + if len(autoIncField) > 0 { + panic(fmt.Sprintf("multiple auto_increment columns: %q, %q", autoIncField, k)) } + autoIncField = k } else { - return fmt.Errorf("fail to insert for null primary key `%s`", k) + return "", nil, fmt.Errorf("cannot insert for null primary key %q", k) } - } else if ! c.IsSupportDefault() && len(c.Default()) > 0 && ov != nil && c.IsZero(ov) { // empty text value + } else if !c.IsSupportDefault() && len(c.Default()) > 0 && ov != nil && c.IsZero(ov) { // empty text value val := c.ConvertFromString(c.Default()) values = append(values, val) names = append(names, fmt.Sprintf("`%s`", k)) @@ -74,6 +63,21 @@ func (t *STableSpec) insert(dt interface{}, debug bool) error { t.name, strings.Join(names, ", "), strings.Join(format, ", ")) + return insertSql, values, nil +} + +func (t *STableSpec) insert(data interface{}, debug bool) error { + beforeInsertFunc := reflect.ValueOf(data).MethodByName("BeforeInsert") + if beforeInsertFunc.IsValid() && !beforeInsertFunc.IsNil() { + beforeInsertFunc.Call([]reflect.Value{}) + } + + dataValue := reflect.ValueOf(data).Elem() + dataFields := reflectutils.FetchStructFieldNameValueInterfaces(dataValue) + insertSql, values, err := t.insertSqlPrep(dataFields) + if err != nil { + return err + } if DEBUG_SQLCHEMY || debug { log.Debugf("%s values: %s", insertSql, values) @@ -92,15 +96,15 @@ func (t *STableSpec) insert(dt interface{}, debug bool) error { } /* - if len(autoIncField) > 0 { - lastId, err := results.LastInsertId() - if err == nil { - val, ok := reflectutils.FindStructFieldValue(dataValue, autoIncField) - if ok { - gotypes.SetValue(val, fmt.Sprint(lastId)) + if len(autoIncField) > 0 { + lastId, err := results.LastInsertId() + if err == nil { + val, ok := reflectutils.FindStructFieldValue(dataValue, autoIncField) + if ok { + gotypes.SetValue(val, fmt.Sprint(lastId)) + } } } - } */ // query the value, so default value can be feedback into the object @@ -112,21 +116,21 @@ func (t *STableSpec) insert(dt interface{}, debug bool) error { if ok && nc.IsAutoIncrement { lastId, err := results.LastInsertId() if err != nil { - log.Errorf("Fail to fetch lastInsertId %s", err) + err := fmt.Errorf("fetching lastInsertId failed: %v", err) return err } else { q = q.Equals(c.Name(), lastId) } } else { - q = q.Equals(c.Name(), fields[c.Name()]) + q = q.Equals(c.Name(), dataFields[c.Name()]) } } } - err = q.First(dt) + err = q.First(data) if err != nil { - log.Errorf("query after insert failed %s", err) + err := fmt.Errorf("query after insert failed: %v", err) return err } - + return nil } diff --git a/vendor/yunion.io/x/sqlchemy/parser.go b/vendor/yunion.io/x/sqlchemy/parser.go index 2464f9c82c..c66f38bf7b 100644 --- a/vendor/yunion.io/x/sqlchemy/parser.go +++ b/vendor/yunion.io/x/sqlchemy/parser.go @@ -3,14 +3,13 @@ package sqlchemy import ( "reflect" - "yunion.io/x/log" "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/tristate" "yunion.io/x/pkg/util/reflectutils" "yunion.io/x/pkg/utils" ) -func fieldToColumnSpec(field *reflect.StructField) IColumnSpec { +func structField2ColumnSpec(field *reflect.StructField) IColumnSpec { fieldname := reflectutils.GetStructFieldName(field) tagmap := utils.TagMap(field.Tag) if _, ok := tagmap[TAG_IGNORE]; ok { @@ -85,13 +84,11 @@ func fieldToColumnSpec(field *reflect.StructField) IColumnSpec { col := NewJSONColumn(fieldname, tagmap) return &col*/ default: - if gotypes.IsSerializable(field.Type) { + if field.Type.Implements(gotypes.ISerializableType) { col := NewCompoundColumn(fieldname, tagmap) return &col - } else { - log.Fatalf("Unsupported type! %s", field.Type) - return nil } + panic("not supported type %s" + field.Type.Name()) } } @@ -101,9 +98,9 @@ func struct2TableSpec(st reflect.Type, table *STableSpec) { if f.Type.Kind() == reflect.Struct && f.Type != gotypes.TimeType { struct2TableSpec(f.Type, table) } else { - coldef := fieldToColumnSpec(&f) - if coldef != nil { - table.columns = append(table.columns, coldef) + column := structField2ColumnSpec(&f) + if column != nil { + table.columns = append(table.columns, column) } } } diff --git a/vendor/yunion.io/x/sqlchemy/query.go b/vendor/yunion.io/x/sqlchemy/query.go index 7b0f332e5b..619240006e 100644 --- a/vendor/yunion.io/x/sqlchemy/query.go +++ b/vendor/yunion.io/x/sqlchemy/query.go @@ -126,7 +126,6 @@ func (sq *SSubQuery) Field(id string, alias ...string) IQueryField { return &sqf } } - log.Errorf("field %s not found for subquery", id) return nil } diff --git a/vendor/yunion.io/x/sqlchemy/reflect.go b/vendor/yunion.io/x/sqlchemy/reflect.go index 7c7753a2ad..c85cb63a0e 100644 --- a/vendor/yunion.io/x/sqlchemy/reflect.go +++ b/vendor/yunion.io/x/sqlchemy/reflect.go @@ -25,7 +25,7 @@ func getStringValue(dat interface{}) string { case gotypes.IntType, gotypes.Int8Type, gotypes.Int16Type, gotypes.Int32Type, gotypes.Int64Type: return fmt.Sprintf("%d", value.Int()) case gotypes.UintType, gotypes.Uint8Type, gotypes.Uint16Type, gotypes.Uint32Type, gotypes.Uint64Type: - return fmt.Sprintf("%u", value.Uint()) + return fmt.Sprintf("%d", value.Uint()) case gotypes.Float32Type, gotypes.Float64Type: return fmt.Sprintf("%f", value.Float()) case gotypes.StringType: @@ -53,15 +53,12 @@ func getStringValue(dat interface{}) string { return string(rawBytes) } default: - if gotypes.IsSerializable(value.Type()) { - is, ok := value.Interface().(gotypes.ISerializable) - if !ok { - log.Errorf("%s fail to convert to JSONObject", value) - } else { - return is.String() - } + serializable, ok := value.Interface().(gotypes.ISerializable) + if !ok { + log.Errorf("cannot convert %v to string", value) + return "" } - log.Errorf("Cannot convert to string %s", value) + return serializable.String() } return "" } @@ -127,15 +124,15 @@ func setValueBySQLString(value reflect.Value, val string) error { gotypes.Float32SliceType, gotypes.Float64SliceType, gotypes.StringSliceType: reflect.Append(value, reflect.ValueOf(val)) default: - if gotypes.IsSerializable(value.Type()) { - is, err := jsonutils.JSONDeserialize(value.Type(), val) - if err != nil { - return err - } - value.Set(reflect.ValueOf(is)) - } else { - return fmt.Errorf("!!!Unsupported type: %s", value.Type) + valueType := value.Type() + if !valueType.Implements(gotypes.ISerializableType) { + return fmt.Errorf("not supported type: %s", valueType) } + serializable, err := jsonutils.JSONDeserialize(valueType, val) + if err != nil { + return err + } + value.Set(reflect.ValueOf(serializable)) } return nil } diff --git a/vendor/yunion.io/x/sqlchemy/sync.go b/vendor/yunion.io/x/sqlchemy/sync.go index 07b3df8a81..d6343129f1 100644 --- a/vendor/yunion.io/x/sqlchemy/sync.go +++ b/vendor/yunion.io/x/sqlchemy/sync.go @@ -212,10 +212,10 @@ func (ts *STableSpec) Sync() error { func (ts *STableSpec) CheckSync() { sqls := ts.SyncSQL() - if sqls != nil && len(sqls) > 0 { + if len(sqls) > 0 { for _, sql := range sqls { fmt.Println(sql) } - log.Fatalf("DB not in sync!!!") + log.Fatalf("DB table %q not in sync", ts.name) } } diff --git a/vendor/yunion.io/x/sqlchemy/table.go b/vendor/yunion.io/x/sqlchemy/table.go index 3f9813f5de..74eedc5a51 100644 --- a/vendor/yunion.io/x/sqlchemy/table.go +++ b/vendor/yunion.io/x/sqlchemy/table.go @@ -5,7 +5,6 @@ import ( "reflect" "strings" - "yunion.io/x/log" "yunion.io/x/pkg/utils" ) @@ -29,12 +28,15 @@ type STableField struct { func NewTableSpecFromStruct(s interface{}, name string) *STableSpec { st := reflect.TypeOf(s) if st.Kind() != reflect.Struct { - log.Fatalf("Invalid table struct, NOT a STRUCT!!!") - return nil + panic("expect Struct kind") } - table := STableSpec{columns: make([]IColumnSpec, 0), name: name, structType: st} - struct2TableSpec(st, &table) - return &table + table := &STableSpec{ + columns: []IColumnSpec{}, + name: name, + structType: st, + } + struct2TableSpec(st, table) + return table } func (ts *STableSpec) Name() string { @@ -88,12 +90,11 @@ func (ts *STableSpec) ColumnSpec(name string) IColumnSpec { func (tbl *STable) Field(name string, alias ...string) IQueryField { // name = reflectutils.StructFieldName(name) name = utils.CamelSplit(name, "_") - cSpec := tbl.spec.ColumnSpec(name) - if cSpec == nil { - log.Fatalf("Column %s not found", name) - return nil + spec := tbl.spec.ColumnSpec(name) + if spec == nil { + panic("column not found: " + name) } - col := STableField{table: tbl, spec: cSpec} + col := STableField{table: tbl, spec: spec} if len(alias) > 0 { col.Label(alias[0]) } diff --git a/vendor/yunion.io/x/sqlchemy/tablealias.go b/vendor/yunion.io/x/sqlchemy/tablealias.go index 47898e4ba6..eec3c89d78 100644 --- a/vendor/yunion.io/x/sqlchemy/tablealias.go +++ b/vendor/yunion.io/x/sqlchemy/tablealias.go @@ -6,14 +6,10 @@ import ( ) var ( - tableID = 0 - tableIDLock *sync.Mutex + tableID = 0 + tableIDLock *sync.Mutex = &sync.Mutex{} ) -func init() { - tableIDLock = &sync.Mutex{} -} - func getTableAliasName() string { tableIDLock.Lock() defer tableIDLock.Unlock() diff --git a/vendor/yunion.io/x/sqlchemy/update.go b/vendor/yunion.io/x/sqlchemy/update.go index b651f338aa..e40e0f6eff 100644 --- a/vendor/yunion.io/x/sqlchemy/update.go +++ b/vendor/yunion.io/x/sqlchemy/update.go @@ -7,6 +7,7 @@ import ( "strings" "yunion.io/x/log" + "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/util/reflectutils" "yunion.io/x/pkg/utils" ) @@ -20,7 +21,6 @@ func (ts *STableSpec) prepareUpdate(dt interface{}) (*SUpdateSession, error) { if reflect.ValueOf(dt).Kind() != reflect.Ptr { return nil, fmt.Errorf("Update input must be a Pointer") } - dataType := reflect.TypeOf(dt).Elem() dataValue := reflect.ValueOf(dt).Elem() fields := reflectutils.FetchStructFieldNameValueInterfaces(dataValue) // fetchStructFieldNameValue(dataType, dataValue) @@ -41,8 +41,7 @@ func (ts *STableSpec) prepareUpdate(dt interface{}) (*SUpdateSession, error) { strings.Join(zeroPrimary, ","), strings.Join(zeroKeyIndex, ",")) } - originValue := reflect.Indirect(reflect.New(dataType)) - originValue.Set(dataValue) + originValue := gotypes.DeepCopyRv(dataValue) us := SUpdateSession{oValue: originValue, tableSpec: ts} return &us, nil } @@ -64,9 +63,9 @@ func UpdateDiffString(diff map[string]SUpdateDiff) string { } func (us *SUpdateSession) saveUpdate(dt interface{}) (map[string]SUpdateDiff, error) { - beforeInsertFunc := reflect.ValueOf(dt).MethodByName("BeforeUpdate") - if beforeInsertFunc.IsValid() && !beforeInsertFunc.IsNil() { - beforeInsertFunc.Call([]reflect.Value{}) + beforeUpdateFunc := reflect.ValueOf(dt).MethodByName("BeforeUpdate") + if beforeUpdateFunc.IsValid() && !beforeUpdateFunc.IsNil() { + beforeUpdateFunc.Call([]reflect.Value{}) } // dataType := reflect.TypeOf(dt).Elem() @@ -127,7 +126,7 @@ func (us *SUpdateSession) saveUpdate(dt interface{}) (map[string]SUpdateDiff, er buf.WriteString(fmt.Sprintf(", `%s` = `%s` + 1", versionField, versionField)) } for _, updatedField := range updatedFields { - buf.WriteString(fmt.Sprintf(", `%s` = NOW()", updatedField)) + buf.WriteString(fmt.Sprintf(", `%s` = UTC_TIMESTAMP()", updatedField)) } buf.WriteString(" WHERE ") first = true diff --git a/vendor/yunion.io/x/structarg/errors.go b/vendor/yunion.io/x/structarg/errors.go new file mode 100644 index 0000000000..3aeeac0b81 --- /dev/null +++ b/vendor/yunion.io/x/structarg/errors.go @@ -0,0 +1,13 @@ +package structarg + +import ( + "fmt" +) + +type NotEnoughArgumentsError struct { + argument Argument +} + +func (e *NotEnoughArgumentsError) Error() string { + return fmt.Sprintf("Not enough arguments, missing %s", e.argument) +} diff --git a/vendor/yunion.io/x/structarg/structarg.go b/vendor/yunion.io/x/structarg/structarg.go index 2f64914771..7f7f6c6414 100644 --- a/vendor/yunion.io/x/structarg/structarg.go +++ b/vendor/yunion.io/x/structarg/structarg.go @@ -27,8 +27,8 @@ type Argument interface { AliasToken() string ShortToken() string MetaVar() string - IsOptional() bool IsPositional() bool + IsRequired() bool IsMulti() bool IsSubcommand() bool HelpString(indent string) string @@ -45,8 +45,8 @@ type SingleArgument struct { aliasToken string shortToken string metavar string - optional bool positional bool + required bool help string choices []string useDefault bool @@ -136,7 +136,13 @@ const ( A boolean value explicitly declare whether the argument is optional, the tag is optional */ - TAG_OPTIONAL = "optional" + TAG_POSITIONAL = "positional" + /* + A boolean value explicitly declare whether the argument is required. + The tag is optional. This is for optional arguments. Positional + arguments must be "required" + */ + TAG_REQUIRED = "required" /* A boolean value explicitly decalre whther the argument is an subcommand A subcommand argument must be the last positional argument. @@ -263,51 +269,67 @@ func (this *ArgumentParser) addArgument(f reflect.StructField, v reflect.Value) } } } - var positional, optional bool + // heuristic guessing "positional" + var positional bool if f.Name == strings.ToUpper(f.Name) { positional = true - optional = false } else { positional = false - optional = true } - opt_val := tagMap[TAG_OPTIONAL] - if len(opt_val) > 0 { - if opt_val == "true" { - optional = true - positional = false - } else if opt_val == "false" { - optional = false + if positionalTag := tagMap[TAG_POSITIONAL]; len(positionalTag) > 0 { + switch positionalTag { + case "true": positional = true - } else { - return fmt.Errorf("Invalid optional value %s, neither true nor false", opt_val) + case "false": + positional = false + default: + return fmt.Errorf("Invalid positional tag %q, neither true nor false", positionalTag) } - // fmt.Println(token, "optional", opt_val, optional, positional) } - if positional && !optional && use_default { - return fmt.Errorf("A positional non-optional argument should not set default value") + required := positional + if requiredTag := tagMap[TAG_REQUIRED]; len(requiredTag) > 0 { + switch requiredTag { + case "true": + required = true + case "false": + required = false + default: + return fmt.Errorf("Invalid required tag %q, neither true nor false", requiredTag) + } } - subcommand, e := strconv.ParseBool(tagMap[TAG_SUBCOMMAND]) - if e != nil { + if positional { + if !required { + return fmt.Errorf("positional %s must not have required:false", token) + } + if use_default { + return fmt.Errorf("positional %s must not have default value", token) + } + } + if !positional && use_default && required { + return fmt.Errorf("non-positional argument with default value should not have required:true set") + } + subcommand, err := strconv.ParseBool(tagMap[TAG_SUBCOMMAND]) + if err != nil { subcommand = false } var defval_t reflect.Value if use_default { - defval_t, e = gotypes.ParseValue(defval, f.Type) - if e != nil { - return e + defval_t, err = gotypes.ParseValue(defval, f.Type) + if err != nil { + return err } } if subcommand { positional = true - optional = false } var arg Argument = nil ovalue := reflect.New(v.Type()).Elem() ovalue.Set(v) sarg := SingleArgument{token: token, shortToken: shorttoken, - optional: optional, positional: positional, - metavar: metavar, help: help, + positional: positional, + required: required, + metavar: metavar, + help: help, choices: choices, useDefault: use_default, aliasToken: alias, @@ -321,7 +343,7 @@ func (this *ArgumentParser) addArgument(f reflect.StructField, v reflect.Value) subcommands: make(map[string]SubcommandArgumentData)} } else if f.Type.Kind() == reflect.Array || f.Type.Kind() == reflect.Slice { var min, max int64 - var e error + var err error nargs := tagMap[TAG_NARGS] if nargs == "*" { min = 0 @@ -333,13 +355,13 @@ func (this *ArgumentParser) addArgument(f reflect.StructField, v reflect.Value) min = 1 max = -1 } else { - min, e = strconv.ParseInt(nargs, 10, 64) - if e == nil { + min, err = strconv.ParseInt(nargs, 10, 64) + if err == nil { max = min } else if positional { min = 1 max = -1 - } else if optional { + } else if !required { min = 0 max = -1 } @@ -349,7 +371,11 @@ func (this *ArgumentParser) addArgument(f reflect.StructField, v reflect.Value) } else { arg = &sarg } - return this.AddArgument(arg) + err = this.AddArgument(arg) + if err != nil { + return fmt.Errorf("AddArgument %s: %v", arg, err) + } + return nil } func (this *ArgumentParser) AddArgument(arg Argument) error { @@ -361,13 +387,25 @@ func (this *ArgumentParser) AddArgument(arg Argument) error { return fmt.Errorf("Cannot append positional argument after an array positional argument") case last_arg.IsSubcommand(): return fmt.Errorf("Cannot append positional argument after a subcommand argument") - case last_arg.IsOptional() && !arg.IsOptional(): - return fmt.Errorf("Cannot append positional argument after an optional positional argument") } } this.posArgs = append(this.posArgs, arg) } else { - this.optArgs = append(this.optArgs, arg) + // Put required at the end and try to be stable + if arg.IsRequired() { + this.optArgs = append(this.optArgs, arg) + } else { + var i int + var opt Argument + for i, opt = range this.optArgs { + if opt.IsRequired() { + break + } + } + this.optArgs = append(this.optArgs, nil) + copy(this.optArgs[i+1:], this.optArgs[i:]) + this.optArgs[i] = arg + } } return nil } @@ -432,12 +470,12 @@ func (this *SingleArgument) ShortToken() string { func (this *SingleArgument) String() string { var start, end byte - if this.IsOptional() { - start = '[' - end = ']' - } else { + if this.IsRequired() { start = '<' end = '>' + } else { + start = '[' + end = ']' } if this.IsPositional() { return fmt.Sprintf("%c%s%c", start, this.MetaVar(), end) @@ -450,8 +488,8 @@ func (this *SingleArgument) String() string { } } -func (this *SingleArgument) IsOptional() bool { - return this.optional +func (this *SingleArgument) IsRequired() bool { + return this.required } func (this *SingleArgument) IsPositional() bool { @@ -519,7 +557,7 @@ func (this *SingleArgument) SetDefault() { } func (this *SingleArgument) Validate() error { - if !this.optional && !this.isSet && !this.useDefault { + if this.required && !this.isSet && !this.useDefault { return fmt.Errorf("Non-optional argument %s not set", this.token) } return nil @@ -818,8 +856,8 @@ func (this *ArgumentParser) ParseArgs(args []string, ignore_unknown bool) error } } } - if err == nil && pos_idx < len(this.posArgs) && !this.posArgs[pos_idx].IsOptional() { - err = fmt.Errorf("Not enough arguments, missing %s", this.posArgs[pos_idx]) + if err == nil && pos_idx < len(this.posArgs) { + err = &NotEnoughArgumentsError{argument: this.posArgs[pos_idx]} } if err == nil { err = this.Validate()