opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-09-21 14:19:49 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'release/2.1.0' of ssh://git.yunion.io/~quxuan/onecloud into feature/qx-server-secgroup

Browse Source
This commit is contained in:
屈轩
2018-08-10 17:18:42 +08:00
parent 02feafc99d fd563734b4
commit b409dbfbf4
1947 changed files with 583709 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Gopkg.lock
Generated
+522 -7
View File
@@ -1,6 +1,22 @@
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
[[projects]]
name = "github.com/DataDog/dd-trace-go"
packages = [
"opentracing",
"tracer",
"tracer/ext"
]
revision = "27617015d45e6cd550b9a7ac7715c37cc2f7d020"
version = "v0.6.1"
[[projects]]
name = "github.com/Shopify/sarama"
packages = ["."]
revision = "35324cf48e33d8260e1c7c18854465a904ade249"
version = "v1.17.0"
[[projects]]
name = "github.com/aliyun/alibaba-cloud-sdk-go"
packages = [
@@ -29,6 +45,11 @@
revision = "3391d3790d23d03408670993e957e8f408993c34"
version = "v1.0.1"
[[projects]]
name = "github.com/apache/thrift"
packages = ["lib/go/thrift"]
revision = "e59b73d3c2bf1c328ccb78e683c0462fa1a473c7"
[[projects]]
branch = "master"
name = "github.com/beorn7/perks"
@@ -47,18 +68,115 @@
revision = "c52492ff1b386e5c0ba5271b5eaad165fab09eca"
version = "v0.2.2"
[[projects]]
name = "github.com/coredns/coredns"
packages = [
"core/dnsserver",
"coremain",
"pb",
"plugin",
"plugin/cache",
"plugin/cache/freq",
"plugin/chaos",
"plugin/debug",
"plugin/dnstap",
"plugin/dnstap/dnstapio",
"plugin/dnstap/msg",
"plugin/dnstap/taprw",
"plugin/errors",
"plugin/etcd/msg",
"plugin/file",
"plugin/file/tree",
"plugin/forward",
"plugin/health",
"plugin/hosts",
"plugin/log",
"plugin/metrics",
"plugin/metrics/vars",
"plugin/nsid",
"plugin/pkg/cache",
"plugin/pkg/dnstest",
"plugin/pkg/dnsutil",
"plugin/pkg/doh",
"plugin/pkg/edns",
"plugin/pkg/fall",
"plugin/pkg/fuzz",
"plugin/pkg/healthcheck",
"plugin/pkg/log",
"plugin/pkg/nonwriter",
"plugin/pkg/parse",
"plugin/pkg/rcode",
"plugin/pkg/replacer",
"plugin/pkg/response",
"plugin/pkg/tls",
"plugin/pkg/trace",
"plugin/pkg/uniq",
"plugin/pkg/up",
"plugin/pkg/upstream",
"plugin/pkg/watch",
"plugin/proxy",
"plugin/reload",
"plugin/test",
"plugin/trace",
"request"
]
revision = "2e322f6e8a54f18c6aef9c25a7c432c291a3d9f7"
version = "v1.2.0"
[[projects]]
name = "github.com/davecgh/go-spew"
packages = ["spew"]
revision = "346938d642f2ec3594ed81d874461961cd0faa76"
version = "v1.1.0"
[[projects]]
branch = "master"
name = "github.com/dnstap/golang-dnstap"
packages = ["."]
revision = "2cf77a2b5e11ac8d0ba3892772ac8e1f7b528344"
[[projects]]
name = "github.com/eapache/go-resiliency"
packages = ["breaker"]
revision = "ea41b0fad31007accc7f806884dcdf3da98b79ce"
version = "v1.1.0"
[[projects]]
branch = "master"
name = "github.com/eapache/go-xerial-snappy"
packages = ["."]
revision = "040cc1a32f578808623071247fdbd5cc43f37f5f"
[[projects]]
name = "github.com/eapache/queue"
packages = ["."]
revision = "44cc805cf13205b55f69e14bcb69867d1ae92f98"
version = "v1.1.0"
[[projects]]
branch = "master"
name = "github.com/farsightsec/golang-framestream"
packages = ["."]
revision = "c06a5734334d9629b3db143d74b47eb94ea68612"
[[projects]]
name = "github.com/fatih/color"
packages = ["."]
revision = "5b77d2a35fb0ede96d138fc9a99f5c9b6aef11b4"
version = "v1.7.0"
[[projects]]
branch = "master"
name = "github.com/flynn/go-shlex"
packages = ["."]
revision = "3f9db97f856818214da2e1057f8ad84803971cff"
[[projects]]
name = "github.com/ghodss/yaml"
packages = ["."]
revision = "0ca9ea5df5451ffdf184b4428c902747c2c11cd7"
version = "v1.0.0"
[[projects]]
branch = "master"
name = "github.com/gin-contrib/sse"
@@ -74,12 +192,27 @@
revision = "d459835d2b077e44f7c9b453505ee29881d5d12d"
version = "v1.2"
[[projects]]
name = "github.com/go-logfmt/logfmt"
packages = ["."]
revision = "390ab7935ee28ec6b286364bba9b4dd6410cb3d5"
version = "v0.3.0"
[[projects]]
name = "github.com/go-sql-driver/mysql"
packages = ["."]
revision = "d523deb1b23d913de5bdada721a6071e71283618"
version = "v1.4.0"
[[projects]]
name = "github.com/gogo/protobuf"
packages = [
"proto",
"sortkeys"
]
revision = "636bf0302bc95575d69441b25a2603156ffdddf1"
version = "v1.1.1"
[[projects]]
name = "github.com/golang-plus/errors"
packages = ["."]
@@ -101,12 +234,79 @@
revision = "a0510c37ef1f49c8290c3a9911d0f7f9ed24d335"
version = "v1.0.0"
[[projects]]
branch = "master"
name = "github.com/golang/glog"
packages = ["."]
revision = "23def4e6c14b4da8ac2ed8007337bc5eb5007998"
[[projects]]
name = "github.com/golang/protobuf"
packages = ["proto"]
packages = [
"proto",
"ptypes",
"ptypes/any",
"ptypes/duration",
"ptypes/timestamp"
]
revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265"
version = "v1.1.0"
[[projects]]
branch = "master"
name = "github.com/golang/snappy"
packages = ["."]
revision = "2e65f85255dbc3072edf28d6b5b8efc472979f5a"
[[projects]]
branch = "master"
name = "github.com/google/btree"
packages = ["."]
revision = "e89373fe6b4a7413d7acd6da1725b83ef713e6e4"
[[projects]]
branch = "master"
name = "github.com/google/gofuzz"
packages = ["."]
revision = "24818f796faf91cd76ec7bddd72458fbced7a6c1"
[[projects]]
name = "github.com/google/uuid"
packages = ["."]
revision = "064e2069ce9c359c118179501254f67d7d37ba24"
version = "0.2"
[[projects]]
name = "github.com/googleapis/gnostic"
packages = [
"OpenAPIv2",
"compiler",
"extensions"
]
revision = "7c663266750e7d82587642f65e60bc4083f1f84e"
version = "v0.2.0"
[[projects]]
branch = "master"
name = "github.com/gregjones/httpcache"
packages = [
".",
"diskcache"
]
revision = "9cad4c3443a7200dd6400aef47183728de563a38"
[[projects]]
branch = "master"
name = "github.com/grpc-ecosystem/grpc-opentracing"
packages = ["go/otgrpc"]
revision = "8e809c8a86450a29b90dcc9efbf062d0fe6d9746"
[[projects]]
name = "github.com/imdario/mergo"
packages = ["."]
revision = "9f23e2d6bd2a77f959b2bf6acdbefd708a83a4a4"
version = "v0.3.6"
[[projects]]
name = "github.com/jinzhu/gorm"
packages = [
@@ -134,6 +334,12 @@
revision = "ab8a2e0c74be9d3be70b3184d9acc634935ded82"
version = "1.1.4"
[[projects]]
branch = "master"
name = "github.com/kr/logfmt"
packages = ["."]
revision = "b84e30acd515aadc4b783ad4ff83aff3299bdfe0"
[[projects]]
name = "github.com/mattn/go-colorable"
packages = ["."]
@@ -170,6 +376,23 @@
packages = ["."]
revision = "9520e82c474b0a04dd04f8a40959027271bab992"
[[projects]]
name = "github.com/mholt/caddy"
packages = [
".",
"caddyfile",
"onevent/hook",
"startupshutdown"
]
revision = "d3f338ddab9ee24b376b5c9c51e202581e2d43ba"
version = "v0.10.11"
[[projects]]
name = "github.com/miekg/dns"
packages = ["."]
revision = "5a2b9fab83ff0f8bfc99684bd5f43a37abe560f1"
version = "v1.0.8"
[[projects]]
name = "github.com/modern-go/concurrent"
packages = ["."]
@@ -188,6 +411,56 @@
packages = ["."]
revision = "9ac6cf4d929b2fa8fd2d2e6dec5bb0feb4f4911d"
[[projects]]
branch = "master"
name = "github.com/opentracing-contrib/go-observer"
packages = ["."]
revision = "a52f2342449246d5bcc273e65cbdcfa5f7d6c63c"
[[projects]]
name = "github.com/opentracing/opentracing-go"
packages = [
".",
"ext",
"log"
]
revision = "1949ddbfd147afd4d964a9f00b24eb291e0e7c38"
version = "v1.0.2"
[[projects]]
name = "github.com/openzipkin/zipkin-go-opentracing"
packages = [
".",
"flag",
"thrift/gen-go/scribe",
"thrift/gen-go/zipkincore",
"types",
"wire"
]
revision = "26cf9707480e6b90e5eff22cf0bbf05319154232"
version = "v0.3.4"
[[projects]]
branch = "master"
name = "github.com/petar/GoLLRB"
packages = ["llrb"]
revision = "53be0d36a84c2a886ca057d34b6aa4468df9ccb4"
[[projects]]
name = "github.com/peterbourgon/diskv"
packages = ["."]
revision = "5f041e8faa004a95c88a202771f4cc3e991971e6"
version = "v2.0.1"
[[projects]]
name = "github.com/pierrec/lz4"
packages = [
".",
"internal/xxh32"
]
revision = "1958fd8fff7f115e79725b1288e0b878b3e06b00"
version = "v2.0.3"
[[projects]]
branch = "master"
name = "github.com/pkg/term"
@@ -202,7 +475,10 @@
[[projects]]
name = "github.com/prometheus/client_golang"
packages = ["prometheus"]
packages = [
"prometheus",
"prometheus/promhttp"
]
revision = "c5b7fccd204277076155f10851dad72b76a49317"
version = "v0.8.0"
@@ -233,6 +509,12 @@
]
revision = "05ee40e3a273f7245e8777337fc7b46e533a9a92"
[[projects]]
branch = "master"
name = "github.com/rcrowley/go-metrics"
packages = ["."]
revision = "e2704e165165ec55d062f5919b4b29494e9fa790"
[[projects]]
name = "github.com/satori/go.uuid"
packages = ["."]
@@ -251,6 +533,12 @@
revision = "3e01752db0189b9157070a0e1668a620f9a85da2"
version = "v1.0.6"
[[projects]]
name = "github.com/spf13/pflag"
packages = ["."]
revision = "583c0c0531f06d5278b7d917446061adc344b5cd"
version = "v1.0.1"
[[projects]]
name = "github.com/stretchr/testify"
packages = ["assert"]
@@ -260,8 +548,7 @@
[[projects]]
name = "github.com/ugorji/go"
packages = ["codec"]
revision = "b4c50a2b199d93b13dc15e78929cfb23bfdf21ab"
version = "v1.1.1"
revision = "f3cacc17c85ecb7f1b6a9e373ee85d1480919868"
[[projects]]
branch = "master"
@@ -279,6 +566,7 @@
revision = "d2fe137411c230927e28dd82b1a3bcb618dbb28d"
[[projects]]
branch = "master"
name = "github.com/yunionio/pkg"
packages = [
"gotypes",
@@ -312,12 +600,13 @@
"util/workqueue",
"utils"
]
revision = "8c5eab6f0b758d3888fd9041419e38a5b1129725"
revision = "607ffde100f37b77f1a90289c06ed6b103325ee8"
[[projects]]
branch = "master"
name = "github.com/yunionio/sqlchemy"
packages = ["."]
revision = "a14641a90b9e5c72868b7a8358d62ea1d83eab53"
revision = "b84d8e2b4313670f24cd50caa5bcc0c5288c16b5"
[[projects]]
branch = "master"
@@ -340,6 +629,25 @@
]
revision = "c126467f60eb25f8f27e5a981f32a87e3965053f"
[[projects]]
branch = "master"
name = "golang.org/x/net"
packages = [
"bpf",
"context",
"http/httpguts",
"http2",
"http2/hpack",
"idna",
"internal/iana",
"internal/socket",
"internal/timeseries",
"ipv4",
"ipv6",
"trace"
]
revision = "f4c29de78a2a91c00474a2e689954305c350adf9"
[[projects]]
branch = "master"
name = "golang.org/x/sys"
@@ -349,12 +657,78 @@
]
revision = "e072cadbbdc8dd3d3ffa82b8b4b9304c261d9311"
[[projects]]
name = "golang.org/x/text"
packages = [
"collate",
"collate/build",
"internal/colltab",
"internal/gen",
"internal/tag",
"internal/triegen",
"internal/ucd",
"language",
"secure/bidirule",
"transform",
"unicode/bidi",
"unicode/cldr",
"unicode/norm",
"unicode/rangetable"
]
revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
version = "v0.3.0"
[[projects]]
branch = "master"
name = "golang.org/x/time"
packages = ["rate"]
revision = "fbb02b2291d28baffd63558aa44b4b56f178d650"
[[projects]]
name = "google.golang.org/appengine"
packages = ["cloudsql"]
revision = "b1f26356af11148e710935ed1ac8a7f5702c7612"
version = "v1.1.0"
[[projects]]
branch = "master"
name = "google.golang.org/genproto"
packages = ["googleapis/rpc/status"]
revision = "daca94659cb50e9f37c1b834680f2e46358f10b0"
[[projects]]
name = "google.golang.org/grpc"
packages = [
".",
"balancer",
"balancer/base",
"balancer/roundrobin",
"codes",
"connectivity",
"credentials",
"encoding",
"encoding/proto",
"grpclog",
"internal",
"internal/backoff",
"internal/channelz",
"internal/envconfig",
"internal/grpcrand",
"internal/transport",
"keepalive",
"metadata",
"naming",
"peer",
"resolver",
"resolver/dns",
"resolver/passthrough",
"stats",
"status",
"tap"
]
revision = "32fb0ac620c32ba40a4626ddf94d90d12cce3455"
version = "v1.14.0"
[[projects]]
name = "gopkg.in/gin-gonic/gin.v1"
packages = ["."]
@@ -367,15 +741,156 @@
revision = "5f1438d3fca68893a817e4a66806cea46a9e4ebf"
version = "v8.18.2"
[[projects]]
name = "gopkg.in/inf.v0"
packages = ["."]
revision = "d2d2541c53f18d2a059457998ce2876cc8e67cbf"
version = "v0.9.1"
[[projects]]
name = "gopkg.in/yaml.v2"
packages = ["."]
revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183"
version = "v2.2.1"
[[projects]]
name = "k8s.io/api"
packages = [
"admissionregistration/v1alpha1",
"admissionregistration/v1beta1",
"apps/v1",
"apps/v1beta1",
"apps/v1beta2",
"authentication/v1",
"authentication/v1beta1",
"authorization/v1",
"authorization/v1beta1",
"autoscaling/v1",
"autoscaling/v2beta1",
"batch/v1",
"batch/v1beta1",
"batch/v2alpha1",
"certificates/v1beta1",
"core/v1",
"events/v1beta1",
"extensions/v1beta1",
"networking/v1",
"policy/v1beta1",
"rbac/v1",
"rbac/v1alpha1",
"rbac/v1beta1",
"scheduling/v1alpha1",
"scheduling/v1beta1",
"settings/v1alpha1",
"storage/v1",
"storage/v1alpha1",
"storage/v1beta1"
]
revision = "072894a440bdee3a891dea811fe42902311cd2a3"
[[projects]]
name = "k8s.io/apimachinery"
packages = [
"pkg/api/errors",
"pkg/api/meta",
"pkg/api/resource",
"pkg/apis/meta/v1",
"pkg/apis/meta/v1/unstructured",
"pkg/apis/meta/v1beta1",
"pkg/conversion",
"pkg/conversion/queryparams",
"pkg/fields",
"pkg/labels",
"pkg/runtime",
"pkg/runtime/schema",
"pkg/runtime/serializer",
"pkg/runtime/serializer/json",
"pkg/runtime/serializer/protobuf",
"pkg/runtime/serializer/recognizer",
"pkg/runtime/serializer/streaming",
"pkg/runtime/serializer/versioning",
"pkg/selection",
"pkg/types",
"pkg/util/clock",
"pkg/util/errors",
"pkg/util/framer",
"pkg/util/intstr",
"pkg/util/json",
"pkg/util/net",
"pkg/util/runtime",
"pkg/util/sets",
"pkg/util/validation",
"pkg/util/validation/field",
"pkg/util/wait",
"pkg/util/yaml",
"pkg/version",
"pkg/watch",
"third_party/forked/golang/reflect"
]
revision = "103fd098999dc9c0c88536f5c9ad2e5da39373ae"
[[projects]]
name = "k8s.io/client-go"
packages = [
"discovery",
"kubernetes",
"kubernetes/scheme",
"kubernetes/typed/admissionregistration/v1alpha1",
"kubernetes/typed/admissionregistration/v1beta1",
"kubernetes/typed/apps/v1",
"kubernetes/typed/apps/v1beta1",
"kubernetes/typed/apps/v1beta2",
"kubernetes/typed/authentication/v1",
"kubernetes/typed/authentication/v1beta1",
"kubernetes/typed/authorization/v1",
"kubernetes/typed/authorization/v1beta1",
"kubernetes/typed/autoscaling/v1",
"kubernetes/typed/autoscaling/v2beta1",
"kubernetes/typed/batch/v1",
"kubernetes/typed/batch/v1beta1",
"kubernetes/typed/batch/v2alpha1",
"kubernetes/typed/certificates/v1beta1",
"kubernetes/typed/core/v1",
"kubernetes/typed/events/v1beta1",
"kubernetes/typed/extensions/v1beta1",
"kubernetes/typed/networking/v1",
"kubernetes/typed/policy/v1beta1",
"kubernetes/typed/rbac/v1",
"kubernetes/typed/rbac/v1alpha1",
"kubernetes/typed/rbac/v1beta1",
"kubernetes/typed/scheduling/v1alpha1",
"kubernetes/typed/scheduling/v1beta1",
"kubernetes/typed/settings/v1alpha1",
"kubernetes/typed/storage/v1",
"kubernetes/typed/storage/v1alpha1",
"kubernetes/typed/storage/v1beta1",
"pkg/apis/clientauthentication",
"pkg/apis/clientauthentication/v1alpha1",
"pkg/apis/clientauthentication/v1beta1",
"pkg/version",
"plugin/pkg/client/auth/exec",
"rest",
"rest/watch",
"tools/auth",
"tools/clientcmd",
"tools/clientcmd/api",
"tools/clientcmd/api/latest",
"tools/clientcmd/api/v1",
"tools/metrics",
"tools/reference",
"transport",
"util/cert",
"util/connrotation",
"util/flowcontrol",
"util/homedir",
"util/integer"
]
revision = "7d04d0e2a0a1a4d4a1cd6baa432a2301492e4e65"
version = "v8.0.0"
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "e90b051392bcbf484678dc55e4b1e44b57442f7b000c6d9956b2979560751ec1"
inputs-digest = "ac275c3d7f14e4a959b3029058a3bb40bb016860ad0e9794e7a396ba33e62048"
solver-name = "gps-cdcl"
solver-version = 1
Gopkg.toml
+30 -2
View File
@@ -34,7 +34,7 @@
name = "github.com/yunionio/jsonutils"
[[constraint]]
revision = "a14641a90b9e5c72868b7a8358d62ea1d83eab53"
branch = "master"
name = "github.com/yunionio/sqlchemy"
[[constraint]]
@@ -42,13 +42,41 @@
name = "github.com/yunionio/log"
[[constraint]]
revision = "8c5eab6f0b758d3888fd9041419e38a5b1129725"
branch = "master"
name = "github.com/yunionio/pkg"
[[constraint]]
branch = "master"
name = "github.com/yunionio/structarg"
# client-go v8.0.0 uses apimachinery 103fd098999dc9c0c88536f5c9ad2e5da39373ae
# and api 072894a440bdee3a891dea811fe42902311cd2a3 (see Godep.json). go dep
# is unable to match Godep.json automatically so have to specify here.
[[constraint]]
name = "k8s.io/client-go"
version = "v8.0.0"
[[override]]
name = "k8s.io/apimachinery"
revision = "103fd098999dc9c0c88536f5c9ad2e5da39373ae"
[[override]]
name = "k8s.io/api"
revision = "072894a440bdee3a891dea811fe42902311cd2a3"
[[override]]
name = "github.com/apache/thrift"
revision = "e59b73d3c2bf1c328ccb78e683c0462fa1a473c7"
[[override]]
name = "github.com/ugorji/go"
revision = "f3cacc17c85ecb7f1b6a9e373ee85d1480919868"
[[constraint]]
name = "github.com/coreos/etcd"
version = "3.3.5"
[[constraint]]
name = "github.com/coredns/coredns"
version = "v1.2.0"
[prune]
go-tests = true
unused-packages = true
Makefile
+11 -1
View File
@@ -3,7 +3,7 @@
REPO_PREFIX := github.com/yunionio/onecloud
VENDOR_PATH := $(REPO_PREFIX)/vendor
VERSION_PKG := $(VENDOR_PATH)/github.com/yunionio/pkg/util/version
ROOT_DIR := $(shell pwd)
ROOT_DIR := $(shell readlink -f `pwd`)
BUILD_DIR := $(ROOT_DIR)/_output
BIN_DIR := $(BUILD_DIR)/bin
BUILD_SCRIPT := $(ROOT_DIR)/build/build.sh
@@ -84,6 +84,16 @@ output_dir:
@mkdir -p $(BUILD_DIR)
dep:
cd $(ROOT_DIR) && dep ensure -v
dep_clean:
rm -fr $(GOPATH)/pkg/dep/sources/*
dep_install:
curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
.PHONY: all build prepare_dir clean fmt rpm
build/build.sh
+21 -8
View File
@@ -2,9 +2,11 @@
set -e
pushd $(dirname $(readlink -f "$BASH_SOURCE")) > /dev/null
ROOT_DIR=$(cd .. && pwd)
popd > /dev/null
if [ -z "$ROOT_DIR" ]; then
pushd $(dirname $(readlink -f "$BASH_SOURCE")) > /dev/null
ROOT_DIR=$(cd .. && pwd)
popd > /dev/null
fi
SRC_BIN=$ROOT_DIR/_output/bin
SRC_BUILD=$ROOT_DIR/build
@@ -38,15 +40,21 @@ echo "Build root ${BUILDROOT}"
# BRANCH=$(git rev-parse --abbrev-ref HEAD)
# TAG=$(git describe --exact-match --tags)
TAG=$(git describe --abbrev=0 --tags)
VERSION=${TAG/\//-}
VERSION=${VERSION/v/}
if [ -z "$VERSION" ]; then
TAG=$(git describe --abbrev=0 --tags || echo testing)
VERSION=${TAG/\//-}
VERSION=${VERSION/v/}
fi
RELEASE=`date +"%y%m%d%H"`
SPEC_DIR=$BUILDROOT/SPECS
SPEC_FILE=$SPEC_DIR/${PKG}.spec
RPM_DIR=$BUILDROOT/RPMS
if [ -z "$OWNER" ]; then
OWNER=yunion
fi
if [ -z "$SERVICE" ]; then
SERVICE="0"
fi
@@ -55,7 +63,7 @@ mkdir -p $SPEC_DIR
echo "# Yunion RPM spec
%global owner yunion
%global owner $OWNER
%global pkgname yunion-$PKG
%global homedir /var/run/%{owner}
%global use_systemd $SERVICE
@@ -82,6 +90,9 @@ $DESCRIPTION
%install
install -D -m 0755 $BIN \$RPM_BUILD_ROOT/opt/yunion/bin/$PKG
for bin in $EXTRA_BINS; do
install -D -m 0755 $SRC_BIN/\$bin \$RPM_BUILD_ROOT/opt/yunion/bin/\$bin
done
if [ -d $ROOT/root ]; then
rsync -a $ROOT/root/ \$RPM_BUILD_ROOT
fi
@@ -112,7 +123,9 @@ getent passwd %{owner} >/dev/null || /usr/sbin/useradd -r -s /sbin/nologin -d %{
%files
%doc
/opt/yunion/bin/$PKG" > $SPEC_FILE
/opt/yunion/bin/$PKG
$(for b in $EXTRA_BINS; do echo /opt/yunion/bin/$b; done)
" > $SPEC_FILE
find $ROOT/root/ -type f | sed -e "s:$ROOT/root::g" >> $SPEC_FILE
build/region-dns/root/etc/yunion/region-dns.conf.sample
+11
View File
@@ -0,0 +1,11 @@
.:53 {
cache 30
yunion . {
sql_connection mysql+pymysql://yunioncloud:AxuaId26ZhfPXWgr@10.168.222.183:3306/yunioncloud?charset=utf8
dns_domain yunion.local
kube_config /home/lzx/.kube/config-hq
fallthrough .
}
proxy . 114.114.114.114:53 8.8.8.8:53
log
}
cmd/climc/shell/hosts.go
+34
View File
@@ -388,6 +388,40 @@ func init() {
return nil
})
type HostEnableNetIfOptions struct {
ID string `help:"ID or Name of host"`
MAC string `help:"MAC of NIC to enable"`
}
R(&HostEnableNetIfOptions{}, "host-enable-netif", "Enable a network interface for a host", func(s *mcclient.ClientSession, args *HostEnableNetIfOptions) error {
params := jsonutils.NewDict()
params.Add(jsonutils.NewString(args.MAC), "mac")
result, err := modules.Hosts.PerformAction(s, args.ID, "enable-netif", params)
if err != nil {
return err
}
printObject(result)
return nil
})
type HostDisableNetIfOptions struct {
ID string `help:"ID or Name of host"`
MAC string `help:"MAC of NIC to disable"`
Reserve bool `help:"Reserve the IP address"`
}
R(&HostDisableNetIfOptions{}, "host-disable-netif", "Disable a network interface", func(s *mcclient.ClientSession, args *HostDisableNetIfOptions) error {
params := jsonutils.NewDict()
params.Add(jsonutils.NewString(args.MAC), "mac")
if args.Reserve {
params.Add(jsonutils.JSONTrue, "reserve")
}
result, err := modules.Hosts.PerformAction(s, args.ID, "disable-netif", params)
if err != nil {
return err
}
printObject(result)
return nil
})
R(&HostDetailOptions{}, "host-remove-all-netifs", "Remvoe all netifs expect admin&ipmi netifs", func(s *mcclient.ClientSession, args *HostDetailOptions) error {
result, err := modules.Hosts.PerformAction(s, args.ID, "remove-all-netifs", nil)
if err != nil {
cmd/climc/shell/k8s.go
+10 -2
View File
@@ -128,8 +128,16 @@ func initCluster() {
return nil
})
R(&getOpt{}, cmdN("kubeconfig"), "Generate kubeconfig of a cluster", func(s *mcclient.ClientSession, args *getOpt) error {
ret, err := k8s.Clusters.PerformAction(s, args.ID, "generate-kubeconfig", nil)
type kubeConfigOpt struct {
getOpt
Directly bool `help:"Get directly connect kubeconfig"`
}
R(&kubeConfigOpt{}, cmdN("kubeconfig"), "Generate kubeconfig of a cluster", func(s *mcclient.ClientSession, args *kubeConfigOpt) error {
params := jsonutils.NewDict()
if args.Directly {
params.Add(jsonutils.JSONTrue, "directly")
}
ret, err := k8s.Clusters.PerformAction(s, args.ID, "generate-kubeconfig", params)
if err != nil {
return err
}
cmd/climc/shell/zones.go
+4
View File
@@ -106,6 +106,7 @@ func init() {
NameCN string `help:"Name in Chinese"`
Desc string `metavar:"<DESCRIPTION>" help:"Description"`
Location string `help:"Location"`
Region string `help:"Cloudregion in which zone created"`
}
R(&ZoneCreateOptions{}, "zone-create", "Create a zone", func(s *mcclient.ClientSession, args *ZoneCreateOptions) error {
params := jsonutils.NewDict()
@@ -119,6 +120,9 @@ func init() {
if len(args.Location) > 0 {
params.Add(jsonutils.NewString(args.Location), "location")
}
if len(args.Region) > 0 {
params.Add(jsonutils.NewString(args.Region), "region")
}
zone, err := modules.Zones.Create(s, params)
if err != nil {
return err
cmd/region-dns/main.go
+49
View File
@@ -0,0 +1,49 @@
package main
import (
_ "github.com/coredns/coredns/plugin/cache"
_ "github.com/coredns/coredns/plugin/chaos"
_ "github.com/coredns/coredns/plugin/debug"
_ "github.com/coredns/coredns/plugin/errors"
_ "github.com/coredns/coredns/plugin/file"
_ "github.com/coredns/coredns/plugin/forward"
_ "github.com/coredns/coredns/plugin/health"
_ "github.com/coredns/coredns/plugin/hosts"
_ "github.com/coredns/coredns/plugin/log"
_ "github.com/coredns/coredns/plugin/metrics"
_ "github.com/coredns/coredns/plugin/nsid"
_ "github.com/coredns/coredns/plugin/proxy"
_ "github.com/coredns/coredns/plugin/reload"
_ "github.com/coredns/coredns/plugin/trace"
_ "github.com/mholt/caddy/startupshutdown"
"github.com/coredns/coredns/core/dnsserver"
"github.com/coredns/coredns/coremain"
_ "github.com/yunionio/onecloud/pkg/dns"
)
var directives = []string{
"debug",
"trace",
"health",
"errors",
"log",
"chaos",
"cache",
"hosts",
"file",
"yunion",
"forward",
"proxy",
"startup",
"shutdown",
}
func init() {
dnsserver.Directives = directives
}
func main() {
coremain.Run()
}
pkg/cloudcommon/db/db_dispatcher.go
+6 -6
View File
@@ -9,18 +9,18 @@ import (
"github.com/yunionio/jsonutils"
"github.com/yunionio/log"
"github.com/yunionio/onecloud/pkg/mcclient"
"github.com/yunionio/onecloud/pkg/mcclient/auth"
"github.com/yunionio/onecloud/pkg/mcclient/modules"
"github.com/yunionio/onecloud/pkg/appsrv"
"github.com/yunionio/pkg/gotypes"
"github.com/yunionio/onecloud/pkg/httperrors"
"github.com/yunionio/pkg/util/filterclause"
"github.com/yunionio/onecloud/pkg/util/httputils"
"github.com/yunionio/pkg/utils"
"github.com/yunionio/sqlchemy"
"github.com/yunionio/onecloud/pkg/appsrv"
"github.com/yunionio/onecloud/pkg/cloudcommon/db/lockman"
"github.com/yunionio/onecloud/pkg/httperrors"
"github.com/yunionio/onecloud/pkg/mcclient"
"github.com/yunionio/onecloud/pkg/mcclient/auth"
"github.com/yunionio/onecloud/pkg/mcclient/modules"
"github.com/yunionio/onecloud/pkg/util/httputils"
)
type DBModelDispatcher struct {
pkg/cloudprovider/resources.go
+4
View File
@@ -5,6 +5,7 @@ import (
"github.com/yunionio/jsonutils"
"github.com/yunionio/onecloud/pkg/mcclient"
"github.com/yunionio/pkg/util/secrules"
)
type ICloudResource interface {
@@ -84,6 +85,7 @@ type ICloudStorage interface {
GetManagerId() string
CreateIDisk(name string, sizeGb int, desc string) (ICloudDisk, error)
GetIDisk(idStr string) (ICloudDisk, error)
}
type ICloudHost interface {
@@ -142,6 +144,7 @@ type ICloudVM interface {
GetBios() string
GetMachine() string
SyncSecurityGroup(secgroupId string, rules []*secrules.SecurityRule) error
GetHypervisor() string
// GetSecurityGroup() ICloudSecurityGroup
@@ -187,6 +190,7 @@ type ICloudDisk interface {
GetDriver() string
GetCacheMode() string
GetMountpoint() string
Delete() error
}
type ICloudVpc interface {
pkg/compute/guestdrivers/aliyun.go
+23 -1
View File
@@ -7,8 +7,8 @@ import (
"github.com/yunionio/jsonutils"
"github.com/yunionio/log"
"github.com/yunionio/onecloud/pkg/mcclient"
"github.com/yunionio/onecloud/pkg/httperrors"
"github.com/yunionio/onecloud/pkg/mcclient"
"github.com/yunionio/pkg/util/seclib"
"github.com/yunionio/pkg/utils"
@@ -163,6 +163,12 @@ func (self *SAliyunGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gu
return nil, err
}
if len(guest.SecgrpId) > 0 {
if err := iVM.SyncSecurityGroup(guest.SecgrpId, guest.GetSecRules()); err != nil {
return nil, err
}
}
if onfinish == "none" {
err = iVM.StartVM()
if err != nil {
@@ -262,3 +268,19 @@ func (self *SAliyunGuestDriver) GetGuestVncInfo(userCred mcclient.TokenCredentia
return dataDict, nil
}
func (self *SAliyunGuestDriver) RequestSyncConfigOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error {
taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {
if fw_only, _ := task.GetParams().Bool("fw_only"); fw_only {
if ihost, err := host.GetIHost(); err != nil {
return nil, err
} else if iVM, err := ihost.GetIVMById(guest.ExternalId); err != nil {
return nil, err
} else if err := iVM.SyncSecurityGroup(guest.SecgrpId, guest.GetSecRules()); err != nil {
return nil, err
}
}
return nil, nil
})
return nil
}
pkg/compute/guestdrivers/baremetals.go
+5
View File
@@ -154,3 +154,8 @@ func (self *SBaremetalGuestDriver) RequestDeployGuestOnHost(ctx context.Context,
return nil
}
func (self *SBaremetalGuestDriver) RequestSyncConfigOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error {
task.ScheduleRun(nil)
return nil
}
pkg/compute/guestdrivers/esxi.go
+8
View File
@@ -1,7 +1,10 @@
package guestdrivers
import (
"context"
"github.com/yunionio/jsonutils"
"github.com/yunionio/onecloud/pkg/cloudcommon/db/taskman"
"github.com/yunionio/onecloud/pkg/mcclient"
"github.com/yunionio/onecloud/pkg/compute/models"
@@ -25,3 +28,8 @@ func (self *SESXiGuestDriver) GetGuestVncInfo(userCred mcclient.TokenCredential,
// TODO
return data, nil
}
func (self *SESXiGuestDriver) RequestSyncConfigOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error {
task.ScheduleRun(nil)
return nil
}
pkg/compute/guestdrivers/kvm.go
+15
View File
@@ -176,3 +176,18 @@ func (self *SKVMGuestDriver) RequestGuestCreateAllDisks(ctx context.Context, gue
// ToDo
return fmt.Errorf("Not Implement")
}
func (self *SKVMGuestDriver) RequestSyncConfigOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error {
desc := guest.GetDriver().GetJsonDescAtHost(ctx, guest, host)
body := jsonutils.NewDict()
body.Add(desc, "desc")
if fw_only, _ := task.GetParams().Bool("fw_only"); fw_only {
body.Add(jsonutils.JSONTrue, "fw_only")
}
url := fmt.Sprintf("/servers/%s/sync", guest.Id)
header := http.Header{}
header.Add("X-Task-Id", task.GetTaskId())
header.Add("X-Region-Version", "v2")
_, err := host.Request(task.GetUserCred(), "POST", url, header, body)
return err
}
pkg/compute/guestdrivers/virtualization.go
+1 -1
View File
@@ -6,8 +6,8 @@ import (
"regexp"
"github.com/yunionio/jsonutils"
"github.com/yunionio/onecloud/pkg/mcclient"
"github.com/yunionio/onecloud/pkg/httperrors"
"github.com/yunionio/onecloud/pkg/mcclient"
"github.com/yunionio/onecloud/pkg/cloudcommon/db/quotas"
"github.com/yunionio/onecloud/pkg/cloudcommon/db/taskman"
pkg/compute/hostdrivers/aliyun.go
+38
View File
@@ -46,3 +46,41 @@ func (self *SAliyunHostDriver) CheckAndSetCacheImage(ctx context.Context, host *
})
return nil
}
func (self *SAliyunHostDriver) RequestAllocateDiskOnStorage(host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask, content *jsonutils.JSONDict) error {
if iCloudStorage, err := storage.GetIStorage(); err != nil {
return err
} else {
if size, err := content.Int("size"); err != nil {
return err
} else {
size = size >> 10
if disk, err := iCloudStorage.CreateIDisk(disk.GetName(), int(size), ""); err != nil {
return err
} else {
data := jsonutils.NewDict()
data.Add(jsonutils.NewInt(int64(disk.GetDiskSizeMB())), "disk_size")
data.Add(jsonutils.NewString(disk.GetDiskFormat()), "disk_format")
task.ScheduleRun(data)
}
}
}
return nil
}
func (self *SAliyunHostDriver) RequestDeallocateDiskOnHost(host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask) error {
if iCloudStorage, err := storage.GetIStorage(); err != nil {
return err
} else if iDisk, err := iCloudStorage.GetIDisk(disk.GetExternalId()); err != nil {
return err
} else {
return iDisk.Delete()
}
}
func (self *SAliyunHostDriver) RequestResizeDiskOnHostOnline(host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error {
return nil
}
func (self *SAliyunHostDriver) RequestResizeDiskOnHost(host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error {
return nil
}
pkg/compute/hostdrivers/kvm.go
+52
View File
@@ -6,6 +6,7 @@ import (
"net/http"
"github.com/yunionio/jsonutils"
"github.com/yunionio/log"
"github.com/yunionio/onecloud/pkg/cloudcommon/db/taskman"
"github.com/yunionio/onecloud/pkg/compute/models"
"github.com/yunionio/onecloud/pkg/util/httputils"
@@ -72,3 +73,54 @@ func (self *SKVMHostDriver) CheckAndSetCacheImage(ctx context.Context, host *mod
}
return nil
}
func (self *SKVMHostDriver) RequestAllocateDiskOnStorage(host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask, content *jsonutils.JSONDict) error {
header := http.Header{}
header.Add("X-Task-Id", task.GetTaskId())
header.Add("X-Region-Version", "v2")
url := fmt.Sprintf("/disks/%s/create/%s", storage.Id, disk.Id)
body := jsonutils.NewDict()
body.Add(content, "disk")
_, err := host.Request(task.GetUserCred(), "POST", url, header, body)
return err
}
func (self *SKVMHostDriver) RequestDeallocateDiskOnHost(host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask) error {
log.Infof("Deallocating disk on host %s", host.GetName())
header := http.Header{}
header.Add("X-Task-Id", task.GetTaskId())
header.Add("X-Region-Version", "v2")
url := fmt.Sprintf("/disks/%s/delete/%s", storage.Id, disk.Id)
body := jsonutils.NewDict()
_, err := host.Request(task.GetUserCred(), "POST", url, header, body)
return err
}
func (self *SKVMHostDriver) RequestResizeDiskOnHost(host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error {
header := http.Header{}
header.Add("X-Task-Id", task.GetTaskId())
header.Add("X-Region-Version", "v2")
url := fmt.Sprintf("/disks/%s/resize/%s", storage.Id, disk.Id)
body := jsonutils.NewDict()
content := jsonutils.NewDict()
content.Add(jsonutils.NewInt(size), "size")
body.Add(content, "disk")
_, err := host.Request(task.GetUserCred(), "POST", url, header, body)
return err
}
func (self *SKVMHostDriver) RequestResizeDiskOnHostOnline(host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error {
self.RequestResizeDiskOnHost(host, storage, disk, size, task)
header := http.Header{}
header.Add("X-Task-Id", task.GetTaskId())
header.Add("X-Region-Version", "v2")
for _, guest := range disk.GetAttachedGuests() {
guestdisk := guest.GetGuestDisk(disk.GetId())
url := fmt.Sprintf("/servers/%s/monitor", guest.GetId())
body := jsonutils.NewDict()
cmd := fmt.Sprintf("block_resize drive_%d %dM", guestdisk.Index, size)
body.Add(jsonutils.NewString(cmd), "cmd")
host.Request(task.GetUserCred(), "POST", url, header, body)
}
return nil
}
pkg/compute/models/disks.go
+208 -7
View File
@@ -9,8 +9,8 @@ import (
"github.com/yunionio/jsonutils"
"github.com/yunionio/log"
"github.com/yunionio/onecloud/pkg/mcclient"
"github.com/yunionio/onecloud/pkg/httperrors"
"github.com/yunionio/sqlchemy"
"github.com/yunionio/pkg/tristate"
"github.com/yunionio/pkg/util/compare"
"github.com/yunionio/pkg/util/fileutils"
@@ -18,12 +18,14 @@ import (
"github.com/yunionio/pkg/util/regutils"
"github.com/yunionio/pkg/util/sysutils"
"github.com/yunionio/pkg/utils"
"github.com/yunionio/sqlchemy"
"github.com/yunionio/onecloud/pkg/cloudcommon/db"
"github.com/yunionio/onecloud/pkg/cloudcommon/db/quotas"
"github.com/yunionio/onecloud/pkg/cloudcommon/db/taskman"
"github.com/yunionio/onecloud/pkg/cloudprovider"
"github.com/yunionio/onecloud/pkg/compute/options"
"github.com/yunionio/onecloud/pkg/httperrors"
"github.com/yunionio/onecloud/pkg/mcclient"
)
const (
@@ -85,6 +87,10 @@ type SDisk struct {
Nonpersistent bool `default:"false" list:"user"` // Column(Boolean, default=False)
}
func (manager *SDiskManager) GetContextManager() []db.IModelManager {
return []db.IModelManager{StorageManager}
}
func (manager *SDiskManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) {
q, err := manager.SSharableVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, query)
if err != nil {
@@ -140,6 +146,10 @@ func (self *SDisk) GetGuestDiskCount() int {
return guestdisks.Equals("disk_id", self.Id).Count()
}
func (self *SDisk) isAttached() bool {
return GuestdiskManager.Query().Equals("disk_id", self.Id).Count() > 0
}
func (self *SDisk) GetGuestdisks() []SGuestdisk {
guestdisks := make([]SGuestdisk, 0)
q := GuestdiskManager.Query().Equals("disk_id", self.Id)
@@ -150,6 +160,190 @@ func (self *SDisk) GetGuestdisks() []SGuestdisk {
}
return guestdisks
}
func (self *SDisk) GetGuests() []SGuest {
result := make([]SGuest, 0)
guests := GuestManager.Query().SubQuery()
guestdisks := GuestdiskManager.Query().SubQuery()
if err := guests.Query().Join(guestdisks, sqlchemy.AND(
sqlchemy.Equals(guestdisks.Field("guest_id"), guests.Field("id")))).
Filter(sqlchemy.Equals(guestdisks.Field("disk_id"), self.Id)).All(&result); err != nil {
log.Errorf(err.Error())
return nil
}
return result
}
func (self *SDisk) GetGuestsCount() int {
guests := GuestManager.Query().SubQuery()
guestdisks := GuestdiskManager.Query().SubQuery()
return guests.Query().Join(guestdisks, sqlchemy.AND(
sqlchemy.Equals(guestdisks.Field("guest_id"), guests.Field("id")))).
Filter(sqlchemy.Equals(guestdisks.Field("disk_id"), self.Id)).Count()
}
func (self *SDisk) GetRuningGuestCount() int {
guests := GuestManager.Query().SubQuery()
guestdisks := GuestdiskManager.Query().SubQuery()
return guests.Query().Join(guestdisks, sqlchemy.AND(
sqlchemy.Equals(guestdisks.Field("guest_id"), guests.Field("id")))).
Filter(sqlchemy.Equals(guestdisks.Field("disk_id"), self.Id)).
Filter(sqlchemy.Equals(guests.Field("status"), VM_RUNNING)).Count()
}
func (self *SDisk) CustomizeCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
diskConfig := SDiskConfig{}
if err := data.Unmarshal(&diskConfig, "disk"); err != nil {
return err
} else {
self.fetchDiskInfo(&diskConfig)
}
return self.SSharableVirtualResourceBase.CustomizeCreate(ctx, userCred, ownerProjId, query, data)
}
func (manager *SDiskManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
if disk, err := data.Get("disk"); err != nil {
return nil, err
} else {
if diskConfig, err := parseDiskInfo(ctx, userCred, disk); err != nil {
return nil, err
} else {
data.Add(jsonutils.Marshal(diskConfig), "disk")
if storageID, err := data.GetString("storage_id"); err != nil {
return nil, err
} else {
storages := StorageManager.Query().SubQuery()
storage := SStorage{}
storage.SetModelManager(StorageManager)
if err := storages.Query().Equals("id", storageID).First(&storage); err != nil {
return nil, err
}
if !storage.Enabled {
return nil, httperrors.NewInputParameterError("Cannot create disk with disabled storage[%s]", storage.Name)
}
if !utils.IsInStringArray(storage.Status, []string{STORAGE_ENABLED, STORAGE_ONLINE}) {
return nil, httperrors.NewInputParameterError("Cannot create disk with offline storage[%s]", storage.Name)
}
if storage.StorageType != diskConfig.Backend {
return nil, httperrors.NewInputParameterError("Storage type[%s] not match backend %s", storage.StorageType, diskConfig.Backend)
}
size := diskConfig.Size >> 10
if storage.StorageType == STORAGE_RBD {
diskConfig.Format = "raw"
data.Add(jsonutils.Marshal(diskConfig), "disk")
} else if storage.StorageType == STORAGE_CLOUD_EFFICIENCY || storage.StorageType == STORAGE_CLOUD_SSD {
if size < 20 || size > 32768 {
return nil, httperrors.NewInputParameterError("cloud_ssd or cloud_efficiency disk only support 20G ~ 32768G")
}
} else if storage.StorageType == STORAGE_PUBLIC_CLOUD {
if size < 5 || size > 2000 {
return nil, httperrors.NewInputParameterError("cloud disk only support 5G ~ 2000G")
}
}
hoststorages := HoststorageManager.Query().SubQuery()
hoststorage := make([]SHoststorage, 0)
if err := hoststorages.Query().Equals("storage_id", storage.Id).All(&hoststorage); err != nil {
return nil, err
}
if len(hoststorage) == 0 {
return nil, httperrors.NewInputParameterError("Storage[%s] must attach to a host", storage.Name)
}
if diskConfig.Size > storage.GetFreeCapacity() {
return nil, httperrors.NewInputParameterError("Not enough free space")
}
if _, err := manager.SSharableVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data); err != nil {
return nil, err
}
pendingUsage := SQuota{Storage: diskConfig.Size}
if err := QuotaManager.CheckSetPendingQuota(ctx, userCred, userCred.GetProjectId(), &pendingUsage); err != nil {
return nil, err
}
}
}
}
return data, nil
}
func (disk *SDisk) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data jsonutils.JSONObject) {
disk.SSharableVirtualResourceBase.PostCreate(ctx, userCred, ownerProjId, query, data)
disk.StartDiskCreateTask(ctx, userCred, false, "", "")
}
func (self *SDisk) StartDiskCreateTask(ctx context.Context, userCred mcclient.TokenCredential, rebuild bool, snapshot string, parentTaskId string) error {
kwargs := jsonutils.NewDict()
if rebuild {
kwargs.Add(jsonutils.JSONFalse, "rebuild")
}
if len(snapshot) > 0 {
kwargs.Add(jsonutils.NewString(snapshot), "snapshot")
}
if task, err := taskman.TaskManager.NewTask(ctx, "DiskCreateTask", self, userCred, kwargs, parentTaskId, "", nil); err != nil {
return err
} else {
task.ScheduleRun(nil)
}
return nil
}
func (self *SDisk) StartAllocate(host *SHost, storage *SStorage, taskId string, userCred mcclient.TokenCredential, rebuild bool, snapshot string, task taskman.ITask) error {
log.Infof("Allocating disk on host %s ...", host.GetName())
templateId := self.GetTemplateId()
fsFormat := self.GetFsFormat()
content := jsonutils.NewDict()
content.Add(jsonutils.NewString(self.DiskFormat), "format")
content.Add(jsonutils.NewInt(int64(self.DiskSize)), "size")
if len(snapshot) > 0 {
content.Add(jsonutils.NewString(snapshot), "snapshot")
} else if len(templateId) > 0 {
content.Add(jsonutils.NewString(templateId), "image_id")
}
if len(fsFormat) > 0 {
content.Add(jsonutils.NewString(fsFormat), "fs_format")
if fsFormat == "ext4" {
name := strings.ToLower(self.GetName())
for _, key := range []string{"encrypt", "secret", "cipher", "private"} {
if strings.Index(key, name) > 0 {
content.Add(jsonutils.JSONTrue, "encryption")
break
}
}
}
}
if rebuild {
content.Add(jsonutils.JSONTrue, "rebuild")
}
return host.GetHostDriver().RequestAllocateDiskOnStorage(host, storage, self, task, content)
}
func (self *SDisk) AllowPerformResize(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
return self.IsOwner(userCred)
}
func (self *SDisk) PerformResize(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
if sizeStr, err := data.GetString("size"); err != nil {
return nil, err
} else if size, err := fileutils.GetSizeMb(sizeStr, 'M', 1024); err != nil {
return nil, err
} else if self.Status != DISK_READY {
return nil, httperrors.NewResourceNotReadyError("Resize disk when disk is READY")
} else if size < self.DiskSize {
return nil, httperrors.NewUnsupportOperationError("Disk cannot be thrink")
} else if size == self.DiskSize {
return nil, nil
} else {
addDisk := size - self.DiskSize
storage := self.GetStorage()
if addDisk > storage.GetFreeCapacity() {
return nil, httperrors.NewOutOfResourceError("Not enough free space")
}
pendingUsage := SQuota{Storage: int(addDisk)}
if err := QuotaManager.CheckSetPendingQuota(ctx, userCred, userCred.GetProjectId(), &pendingUsage); err != nil {
return nil, httperrors.NewOutOfQuotaError(err.Error())
}
return nil, self.StartDiskResizeTask(ctx, userCred, int64(size), "", &pendingUsage)
}
}
func (self *SDisk) ValidateDeleteCondition(ctx context.Context) error {
if self.GetGuestDiskCount() > 0 {
@@ -158,10 +352,6 @@ func (self *SDisk) ValidateDeleteCondition(ctx context.Context) error {
return self.SSharableVirtualResourceBase.ValidateDeleteCondition(ctx)
}
func (self *SDisk) StartAllocate(host *SHost, Storage *SStorage, taskId string, userCred mcclient.TokenCredential, rebuild bool) {
}
func (self *SDisk) GetTemplateId() string {
return self.TemplateId
}
@@ -534,6 +724,17 @@ func (self *SDisk) CustomizeDelete(ctx context.Context, userCred mcclient.TokenC
return self.StartDiskDeleteTask(ctx, userCred, "", false)
}
func (self *SDisk) StartDiskResizeTask(ctx context.Context, userCred mcclient.TokenCredential, size int64, parentTaskId string, pendingUsage quotas.IQuota) error {
params := jsonutils.NewDict()
params.Add(jsonutils.NewInt(size), "size")
if task, err := taskman.TaskManager.NewTask(ctx, "DiskResizeTask", self, userCred, params, parentTaskId, "", pendingUsage); err != nil {
return err
} else {
task.ScheduleRun(nil)
}
return nil
}
func (self *SDisk) StartDiskDeleteTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string, isPurge bool) error {
params := jsonutils.NewDict()
if isPurge {
pkg/compute/models/dnsrecords.go
+2 -2
View File
@@ -7,12 +7,12 @@ import (
"github.com/yunionio/jsonutils"
"github.com/yunionio/log"
"github.com/yunionio/onecloud/pkg/mcclient"
"github.com/yunionio/onecloud/pkg/httperrors"
"github.com/yunionio/pkg/util/regutils"
"github.com/yunionio/sqlchemy"
"github.com/yunionio/onecloud/pkg/cloudcommon/db"
"github.com/yunionio/onecloud/pkg/httperrors"
"github.com/yunionio/onecloud/pkg/mcclient"
)
type SDnsRecordManager struct {
pkg/compute/models/guestdisks.go
+20 -1
View File
@@ -6,9 +6,11 @@ import (
"github.com/yunionio/jsonutils"
"github.com/yunionio/log"
"github.com/yunionio/onecloud/pkg/mcclient"
"github.com/yunionio/sqlchemy"
"github.com/yunionio/onecloud/pkg/cloudcommon/db"
"github.com/yunionio/onecloud/pkg/httperrors"
"github.com/yunionio/onecloud/pkg/mcclient"
)
type SGuestdiskManager struct {
@@ -47,6 +49,23 @@ func (self *SGuestdisk) AllowDeleteItem(ctx context.Context, userCred mcclient.T
return false
}
func (self *SGuestdisk) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
if data.Contains("index") {
if index, err := data.Int("index"); err != nil {
return nil, err
} else {
guestdisk := GuestdiskManager.Query().SubQuery()
count := guestdisk.Query().Filter(sqlchemy.Equals(guestdisk.Field("guest_id"), self.GuestId)).
Filter(sqlchemy.NotEquals(guestdisk.Field("disk_id"), self.DiskId)).
Filter(sqlchemy.Equals(guestdisk.Field("index"), index)).Count()
if count > 0 {
return nil, httperrors.NewInputParameterError("DISK Index %d has been occupied", index)
}
}
}
return self.SGuestJointsBase.ValidateUpdateData(ctx, userCred, query, data)
}
func (joint *SGuestdisk) Master() db.IStandaloneModel {
return db.JointMaster(joint)
}
pkg/compute/models/guestdrivers.go
+3 -1
View File
@@ -5,9 +5,9 @@ import (
"github.com/yunionio/jsonutils"
"github.com/yunionio/log"
"github.com/yunionio/onecloud/pkg/mcclient"
"github.com/yunionio/onecloud/pkg/cloudcommon/db/quotas"
"github.com/yunionio/onecloud/pkg/cloudcommon/db/taskman"
"github.com/yunionio/onecloud/pkg/mcclient"
)
type IGuestDriver interface {
@@ -48,6 +48,8 @@ type IGuestDriver interface {
StartGuestSyncstatusTask(guest *SGuest, ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error
RequestSyncConfigOnHost(ctx context.Context, guest *SGuest, host *SHost, task taskman.ITask) error
RequestSyncstatusOnHost(ctx context.Context, guest *SGuest, host *SHost, userCred mcclient.TokenCredential) (jsonutils.JSONObject, error)
RequestStartOnHost(ctx context.Context, guest *SGuest, host *SHost, userCred mcclient.TokenCredential, task taskman.ITask) (jsonutils.JSONObject, error)
pkg/compute/models/guests.go
+175 -23
View File
@@ -11,6 +11,18 @@ import (
"github.com/yunionio/jsonutils"
"github.com/yunionio/log"
"github.com/yunionio/pkg/tristate"
"github.com/yunionio/pkg/util/compare"
"github.com/yunionio/pkg/util/fileutils"
"github.com/yunionio/pkg/util/netutils"
"github.com/yunionio/pkg/util/osprofile"
"github.com/yunionio/pkg/util/regutils"
"github.com/yunionio/pkg/util/secrules"
"github.com/yunionio/pkg/util/sysutils"
"github.com/yunionio/pkg/util/timeutils"
"github.com/yunionio/pkg/utils"
"github.com/yunionio/sqlchemy"
"github.com/yunionio/onecloud/pkg/cloudcommon/db"
"github.com/yunionio/onecloud/pkg/cloudcommon/db/lockman"
"github.com/yunionio/onecloud/pkg/cloudcommon/db/quotas"
@@ -21,15 +33,6 @@ import (
"github.com/yunionio/onecloud/pkg/httperrors"
"github.com/yunionio/onecloud/pkg/mcclient"
"github.com/yunionio/onecloud/pkg/mcclient/auth"
"github.com/yunionio/pkg/tristate"
"github.com/yunionio/pkg/util/compare"
"github.com/yunionio/pkg/util/fileutils"
"github.com/yunionio/pkg/util/osprofile"
"github.com/yunionio/pkg/util/regutils"
"github.com/yunionio/pkg/util/sysutils"
"github.com/yunionio/pkg/util/timeutils"
"github.com/yunionio/pkg/utils"
"github.com/yunionio/sqlchemy"
)
const (
@@ -896,6 +899,8 @@ func (self *SGuest) GetExtraDetails(ctx context.Context, userCred mcclient.Token
if zone != nil {
extra.Add(jsonutils.NewString(zone.GetId()), "zone_id")
extra.Add(jsonutils.NewString(zone.GetName()), "zone")
extra.Add(jsonutils.NewString(zone.GetRegion().GetName()), "region")
extra.Add(jsonutils.NewString(zone.GetRegion().GetId()), "region_id")
}
return extra
}
@@ -995,6 +1000,16 @@ func (self *SGuest) getRealIPs() []string {
return ips
}
func (self *SGuest) IsExitOnly() bool {
for _, ip := range self.getRealIPs() {
addr, _ := netutils.NewIPV4Addr(ip)
if !netutils.IsExitAddress(addr) {
return false
}
}
return true
}
func (self *SGuest) getVirtualIPs() []string {
ips := make([]string, 0)
for _, guestgroup := range self.GetGroups() {
@@ -1070,6 +1085,22 @@ func (self *SGuest) getAdminSecgroupName() string {
return ""
}
func (self *SGuest) GetSecRules() []*secrules.SecurityRule {
return self.getSecRules()
}
func (self *SGuest) getSecRules() []*secrules.SecurityRule {
if secgrp := self.getSecgroup(); secgrp != nil {
return secgrp.getSecRules()
}
if rule, err := secrules.ParseSecurityRule(options.Options.DefaultSecurityRules); err == nil {
return []*secrules.SecurityRule{rule}
} else {
log.Errorf("Default SecurityRules error: %v", err)
}
return []*secrules.SecurityRule{}
}
func (self *SGuest) getSecurityRules() string {
secgrp := self.getSecgroup()
if secgrp != nil {
@@ -1395,6 +1426,78 @@ func (self *SGuest) attach2Disk(disk *SDisk, userCred mcclient.TokenCredential,
return err
}
func (self *SGuest) AllowPerformSync(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
return self.IsOwner(userCred)
}
func (self *SGuest) PerformSync(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
if err := self.StartSyncTask(ctx, userCred, false, ""); err != nil {
return nil, err
}
return nil, nil
}
func (self *SGuest) AllowPerformAttachDisk(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
return self.IsOwner(userCred)
}
func (self *SGuest) ValidateAttachDisk(ctx context.Context, disk *SDisk) error {
if disk.isAttached() {
return httperrors.NewInputParameterError("Disk %s has been attached", disk.Name)
} else if len(disk.GetPathAtHost(self.GetHost())) == 0 {
return httperrors.NewInputParameterError("Disk %s not belong the guest's host", disk.Name)
} else if disk.Status != DISK_READY {
return httperrors.NewInputParameterError("Disk in %s not able to attach", disk.Status)
} else if !utils.IsInStringArray(self.Status, []string{VM_RUNNING, VM_READY}) {
return httperrors.NewInputParameterError("Server in %s not able to attach disk", self.Status)
}
return nil
}
func (self *SGuest) PerformAttachDisk(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
if diskId, err := data.GetString("disk_id"); err != nil {
return nil, err
} else {
if disk, err := DiskManager.FetchByIdOrName(userCred.GetProjectId(), diskId); err != nil {
return nil, err
} else if disk == nil {
return nil, httperrors.NewResourceNotFoundError("Disk %s not found", diskId)
} else if err := self.ValidateAttachDisk(ctx, disk.(*SDisk)); err != nil {
return nil, err
} else {
driver, _ := data.GetString("driver")
cache, _ := data.GetString("cache")
mountpoint, _ := data.GetString("mountpoint")
if err := self.attach2Disk(disk.(*SDisk), userCred, driver, cache, mountpoint); err != nil {
return nil, err
} else {
self.StartSyncTask(ctx, userCred, false, "")
}
}
}
return nil, nil
}
func (self *SGuest) StartSyncTask(ctx context.Context, userCred mcclient.TokenCredential, fw_only bool, parentTaskId string) error {
if !utils.IsInStringArray(self.Status, []string{VM_READY, VM_RUNNING}) {
return httperrors.NewResourceBusyError("Cannot sync in status %s", self.Status)
}
data := jsonutils.NewDict()
if fw_only {
data.Add(jsonutils.JSONTrue, "fw_only")
} else if err := self.SetStatus(userCred, VM_SYNC_CONFIG, ""); err != nil {
log.Errorf(err.Error())
return err
}
if task, err := taskman.TaskManager.NewTask(ctx, "GuestSyncConfTask", self, userCred, data, parentTaskId, "", nil); err != nil {
log.Errorf(err.Error())
return err
} else {
task.ScheduleRun(nil)
}
return nil
}
type sSyncDiskPair struct {
disk *SDisk
vdisk cloudprovider.ICloudDisk
@@ -1931,9 +2034,8 @@ func (self *SGuest) PerformRevokeSecgroup(ctx context.Context, userCred mcclient
}); err != nil {
return nil, err
}
if self.Status == VM_RUNNING {
//quxuan TODO
//self.StartSyncTask()
if err := self.StartSyncTask(ctx, userCred, true, ""); err != nil {
return nil, err
}
}
return nil, nil
@@ -1954,9 +2056,8 @@ func (self *SGuest) PerformAssignSecgroup(ctx context.Context, userCred mcclient
}); err != nil {
return nil, err
}
if self.Status == VM_RUNNING {
//quxuan TODO
//self.StartSyncTask()
if err := self.StartSyncTask(ctx, userCred, true, ""); err != nil {
return nil, err
}
}
}
@@ -1981,10 +2082,9 @@ func (self *SGuest) PerformAssignSecgroup(ctx context.Context, userCred mcclient
// }); err != nil {
// return nil, err
// }
// if self.Status == VM_RUNNING {
// //quxuan TODO
// //self.StartSyncTask()
// }
// if err := self.StartSyncTask(ctx, userCred, true, ""); err != nil {
// return nil, err
// }
// }
// return nil, nil
// }
@@ -2004,10 +2104,9 @@ func (self *SGuest) PerformAssignSecgroup(ctx context.Context, userCred mcclient
// }); err != nil {
// return nil, err
// }
// if self.Status == VM_RUNNING {
// //quxuan TODO
// //self.StartSyncTask()
// }
// if err := self.StartSyncTask(ctx, userCred, true, ""); err != nil {
// return nil, err
// }
// }
// }
// return nil, nil
@@ -2665,3 +2764,56 @@ func (self *SGuest) GetKeypairPublicKey() string {
func (model *SGuest) AllowPerformCancelDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
return userCred.IsSystemAdmin()
}
func (manager *SGuestManager) GetIpInProjectWithName(projectId, name string, isExitOnly bool) []string {
guestnics := GuestnetworkManager.Query().SubQuery()
guests := manager.Query().SubQuery()
networks := NetworkManager.Query().SubQuery()
q := guestnics.Query(guestnics.Field("ip_addr")).Join(guests,
sqlchemy.AND(
sqlchemy.Equals(guests.Field("id"), guestnics.Field("guest_id")),
sqlchemy.OR(sqlchemy.IsNull(guests.Field("pending_deleted")),
sqlchemy.IsFalse(guests.Field("pending_deleted"))),
sqlchemy.IsFalse(guests.Field("deleted")))).
Join(networks, sqlchemy.AND(sqlchemy.Equals(networks.Field("id"), guestnics.Field("network_id")),
sqlchemy.IsFalse(networks.Field("deleted")))).
Filter(sqlchemy.Equals(guests.Field("name"), name)).
Filter(sqlchemy.NotEquals(guestnics.Field("ip_addr"), "")).
Filter(sqlchemy.IsNotNull(guestnics.Field("ip_addr"))).
Filter(sqlchemy.IsNotNull(networks.Field("guest_gateway")))
ips := make([]string, 0)
rows, err := q.Rows()
if err != nil {
log.Errorf("Get guest ip with name query err: %v", err)
return ips
}
for rows.Next() {
var ip string
err = rows.Scan(&ip)
if err != nil {
log.Errorf("Get guest ip with name scan err: %v", err)
return ips
}
ips = append(ips, ip)
}
return manager.getIpsByExit(ips, isExitOnly)
}
func (manager *SGuestManager) getIpsByExit(ips []string, isExitOnly bool) []string {
intRet := make([]string, 0)
extRet := make([]string, 0)
for _, ip := range ips {
addr, _ := netutils.NewIPV4Addr(ip)
if netutils.IsExitAddress(addr) {
extRet = append(extRet, ip)
continue
}
intRet = append(intRet, ip)
}
if isExitOnly {
return extRet
} else if len(intRet) > 0 {
return intRet
}
return extRet
}
pkg/compute/models/hostdrivers.go
+5
View File
@@ -3,6 +3,7 @@ package models
import (
"context"
"github.com/yunionio/jsonutils"
"github.com/yunionio/log"
"github.com/yunionio/onecloud/pkg/cloudcommon/db/taskman"
)
@@ -10,6 +11,10 @@ import (
type IHostDriver interface {
GetHostType() string
CheckAndSetCacheImage(ctx context.Context, host *SHost, storagecache *SStoragecache, scimg *SStoragecachedimage, task taskman.ITask) error
RequestAllocateDiskOnStorage(host *SHost, storage *SStorage, disk *SDisk, task taskman.ITask, content *jsonutils.JSONDict) error
RequestDeallocateDiskOnHost(host *SHost, storage *SStorage, disk *SDisk, task taskman.ITask) error
RequestResizeDiskOnHostOnline(host *SHost, storage *SStorage, disk *SDisk, size int64, task taskman.ITask) error
RequestResizeDiskOnHost(host *SHost, storage *SStorage, disk *SDisk, size int64, task taskman.ITask) error
}
var hostDrivers map[string]IHostDriver
pkg/compute/models/hostnetworks.go
+47 -16
View File
@@ -4,9 +4,10 @@ import (
"context"
"github.com/yunionio/jsonutils"
"github.com/yunionio/onecloud/pkg/mcclient"
"github.com/yunionio/sqlchemy"
"github.com/yunionio/onecloud/pkg/cloudcommon/db"
"github.com/yunionio/onecloud/pkg/mcclient"
)
type SHostnetworkManager struct {
@@ -31,27 +32,27 @@ type SHostnetwork struct {
MacAddr string `width:"18" charset:"ascii" list:"admin"` // Column(VARCHAR(18, charset='ascii'))
}
func (joint *SHostnetwork) Master() db.IStandaloneModel {
return db.JointMaster(joint)
func (bn *SHostnetwork) Master() db.IStandaloneModel {
return db.JointMaster(bn)
}
func (joint *SHostnetwork) Slave() db.IStandaloneModel {
return db.JointSlave(joint)
func (bn *SHostnetwork) Slave() db.IStandaloneModel {
return db.JointSlave(bn)
}
func (self *SHostnetwork) GetCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict {
extra := self.SHostJointsBase.GetCustomizeColumns(ctx, userCred, query)
extra = db.JointModelExtra(self, extra)
netif := self.GetNetInterface()
func (bn *SHostnetwork) GetCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict {
extra := bn.SHostJointsBase.GetCustomizeColumns(ctx, userCred, query)
extra = db.JointModelExtra(bn, extra)
netif := bn.GetNetInterface()
if netif != nil {
extra.Add(jsonutils.NewString(netif.NicType), "nic_type")
}
return extra
}
func (self *SHostnetwork) GetExtraDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict {
extra := self.SHostJointsBase.GetExtraDetails(ctx, userCred, query)
return db.JointModelExtra(self, extra)
func (bn *SHostnetwork) GetExtraDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict {
extra := bn.SHostJointsBase.GetExtraDetails(ctx, userCred, query)
return db.JointModelExtra(bn, extra)
}
func (bn *SHostnetwork) GetHost() *SHost {
@@ -75,10 +76,40 @@ func (bn *SHostnetwork) GetNetInterface() *SNetInterface {
return netIf
}
func (self *SHostnetwork) Delete(ctx context.Context, userCred mcclient.TokenCredential) error {
return db.DeleteModel(ctx, userCred, self)
func (bn *SHostnetwork) Delete(ctx context.Context, userCred mcclient.TokenCredential) error {
return db.DeleteModel(ctx, userCred, bn)
}
func (self *SHostnetwork) Detach(ctx context.Context, userCred mcclient.TokenCredential) error {
return db.DetachJoint(ctx, userCred, self)
func (bn *SHostnetwork) Detach(ctx context.Context, userCred mcclient.TokenCredential) error {
return db.DetachJoint(ctx, userCred, bn)
}
func (man *SHostnetworkManager) QueryByAddress(addr string) *sqlchemy.SQuery {
q := HostnetworkManager.Query()
return q.Filter(sqlchemy.Equals(q.Field("ip_addr"), addr))
}
func (man *SHostnetworkManager) GetHostNetworkByAddress(addr string) *SHostnetwork {
network := SHostnetwork{}
err := man.QueryByAddress(addr).First(&network)
if err != nil {
return &network
}
return nil
}
func (man *SHostnetworkManager) GetNetworkByAddress(addr string) *SNetwork {
net := man.GetHostNetworkByAddress(addr)
if net == nil {
return nil
}
return net.GetNetwork()
}
func (man *SHostnetworkManager) GetHostByAddress(addr string) *SHost {
net := man.GetHostNetworkByAddress(addr)
if net == nil {
return nil
}
return net.GetHost()
}
pkg/compute/models/hosts.go
+11 -8
View File
@@ -9,13 +9,6 @@ import (
"github.com/yunionio/jsonutils"
"github.com/yunionio/log"
"github.com/yunionio/onecloud/pkg/mcclient"
"github.com/yunionio/onecloud/pkg/mcclient/auth"
"github.com/yunionio/onecloud/pkg/mcclient/modules"
"github.com/yunionio/onecloud/pkg/cloudcommon/db"
"github.com/yunionio/onecloud/pkg/cloudprovider"
"github.com/yunionio/onecloud/pkg/compute/options"
"github.com/yunionio/onecloud/pkg/httperrors"
"github.com/yunionio/pkg/tristate"
"github.com/yunionio/pkg/util/compare"
"github.com/yunionio/pkg/util/netutils"
@@ -23,6 +16,14 @@ import (
"github.com/yunionio/pkg/util/sysutils"
"github.com/yunionio/pkg/utils"
"github.com/yunionio/sqlchemy"
"github.com/yunionio/onecloud/pkg/cloudcommon/db"
"github.com/yunionio/onecloud/pkg/cloudprovider"
"github.com/yunionio/onecloud/pkg/compute/options"
"github.com/yunionio/onecloud/pkg/httperrors"
"github.com/yunionio/onecloud/pkg/mcclient"
"github.com/yunionio/onecloud/pkg/mcclient/auth"
"github.com/yunionio/onecloud/pkg/mcclient/modules"
)
const (
@@ -1116,7 +1117,7 @@ func (self *SHost) SyncHostVMs(ctx context.Context, userCred mcclient.TokenCrede
}
func (self *SHost) getNetworkOfIPOnHost(ipAddr string) (*SNetwork, error) {
net, err := NetworkManager.getNetworkOfIP(ipAddr, "", tristate.None)
net, err := NetworkManager.GetNetworkOfIP(ipAddr, "", tristate.None)
if err != nil {
return nil, err
}
@@ -1438,6 +1439,8 @@ func (self *SHost) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict
if zone != nil {
extra.Add(jsonutils.NewString(zone.Id), "zone_id")
extra.Add(jsonutils.NewString(zone.Name), "zone")
extra.Add(jsonutils.NewString(zone.GetRegion().GetName()), "region")
extra.Add(jsonutils.NewString(zone.GetRegion().GetId()), "region_id")
}
server := self.getBaremetalServer()
if server != nil {
pkg/compute/models/networks.go
+8 -7
View File
@@ -8,12 +8,6 @@ import (
"github.com/yunionio/jsonutils"
"github.com/yunionio/log"
"github.com/yunionio/onecloud/pkg/mcclient"
"github.com/yunionio/onecloud/pkg/cloudcommon/db"
"github.com/yunionio/onecloud/pkg/cloudcommon/db/taskman"
"github.com/yunionio/onecloud/pkg/cloudprovider"
"github.com/yunionio/onecloud/pkg/compute/options"
"github.com/yunionio/onecloud/pkg/httperrors"
"github.com/yunionio/pkg/tristate"
"github.com/yunionio/pkg/util/compare"
"github.com/yunionio/pkg/util/fileutils"
@@ -21,6 +15,13 @@ import (
"github.com/yunionio/pkg/util/regutils"
"github.com/yunionio/pkg/utils"
"github.com/yunionio/sqlchemy"
"github.com/yunionio/onecloud/pkg/cloudcommon/db"
"github.com/yunionio/onecloud/pkg/cloudcommon/db/taskman"
"github.com/yunionio/onecloud/pkg/cloudprovider"
"github.com/yunionio/onecloud/pkg/compute/options"
"github.com/yunionio/onecloud/pkg/httperrors"
"github.com/yunionio/onecloud/pkg/mcclient"
)
const (
@@ -507,7 +508,7 @@ func (self *SNetwork) isAddressUsed(address string) bool {
return false
}
func (manager *SNetworkManager) getNetworkOfIP(ipAddr string, serverType string, isPublic tristate.TriState) (*SNetwork, error) {
func (manager *SNetworkManager) GetNetworkOfIP(ipAddr string, serverType string, isPublic tristate.TriState) (*SNetwork, error) {
address, err := netutils.NewIPV4Addr(ipAddr)
if err != nil {
return nil, err
pkg/compute/models/secgrouprules.go
+4 -4
View File
@@ -8,9 +8,9 @@ import (
"github.com/yunionio/jsonutils"
"github.com/yunionio/log"
"github.com/yunionio/onecloud/pkg/mcclient"
"github.com/yunionio/onecloud/pkg/cloudcommon/db"
"github.com/yunionio/onecloud/pkg/httperrors"
"github.com/yunionio/onecloud/pkg/mcclient"
"github.com/yunionio/pkg/util/secrules"
"github.com/yunionio/pkg/util/stringutils"
"github.com/yunionio/sqlchemy"
@@ -212,7 +212,7 @@ func (self *SSecurityGroupRule) PostCreate(ctx context.Context, userCred mcclien
log.Debugf("POST Create %s", data)
if secgroup := self.GetSecGroup(); secgroup != nil {
secgroup.DoSync()
secgroup.DoSync(ctx, userCred)
}
}
@@ -220,7 +220,7 @@ func (self *SSecurityGroupRule) PreDelete(ctx context.Context, userCred mcclient
self.SResourceBase.PreDelete(ctx, userCred)
if secgroup := self.GetSecGroup(); secgroup != nil {
secgroup.DoSync()
secgroup.DoSync(ctx, userCred)
}
}
@@ -229,6 +229,6 @@ func (self *SSecurityGroupRule) PostUpdate(ctx context.Context, userCred mcclien
log.Debugf("POST Update %s", data)
if secgroup := self.GetSecGroup(); secgroup != nil {
secgroup.DoSync()
secgroup.DoSync(ctx, userCred)
}
}
pkg/compute/models/secgroups.go
+16 -2
View File
@@ -6,9 +6,10 @@ import (
"github.com/yunionio/jsonutils"
"github.com/yunionio/log"
"github.com/yunionio/onecloud/pkg/mcclient"
"github.com/yunionio/onecloud/pkg/cloudcommon/db"
"github.com/yunionio/onecloud/pkg/httperrors"
"github.com/yunionio/onecloud/pkg/mcclient"
"github.com/yunionio/pkg/util/secrules"
"github.com/yunionio/sqlchemy"
)
@@ -99,6 +100,16 @@ func (self *SSecurityGroup) getSecurityRules() (rules []SSecurityGroupRule) {
return
}
func (self *SSecurityGroup) getSecRules() []*secrules.SecurityRule {
rules := make([]*secrules.SecurityRule, 0)
for _, rule := range self.getSecurityRules() {
r, _ := secrules.ParseSecurityRule(rule.GetRule())
r.Priority = int(rule.Priority)
rules = append(rules, r)
}
return rules
}
func (self *SSecurityGroup) getSecurityRuleString() string {
secgrouprules := self.getSecurityRules()
var rules []string
@@ -158,11 +169,14 @@ func (self *SSecurityGroup) PerformClone(ctx context.Context, userCred mcclient.
return nil, nil
}
func (self *SSecurityGroup) DoSync() {
func (self *SSecurityGroup) DoSync(ctx context.Context, userCred mcclient.TokenCredential) {
if _, err := self.GetModelManager().TableSpec().Update(self, func() error {
self.IsDirty = true
return nil
}); err != nil {
log.Errorf("Update Security Group error: %s", err.Error())
}
for _, guest := range self.GetGuests() {
guest.StartSyncTask(ctx, userCred, true, "")
}
}
pkg/compute/models/storages.go
+11 -8
View File
@@ -16,14 +16,17 @@ import (
)
const (
STORAGE_LOCAL = "local"
STORAGE_BAREMETAL = "baremetal"
STORAGE_SHEEPDOG = "sheepdog"
STORAGE_RBD = "rbd"
STORAGE_DOCKER = "docker"
STORAGE_NAS = "nas"
STORAGE_VSAN = "vsan"
STORAGE_PUBLIC_CLOUD = "cloud"
STORAGE_LOCAL = "local"
STORAGE_BAREMETAL = "baremetal"
STORAGE_SHEEPDOG = "sheepdog"
STORAGE_RBD = "rbd"
STORAGE_DOCKER = "docker"
STORAGE_NAS = "nas"
STORAGE_VSAN = "vsan"
STORAGE_PUBLIC_CLOUD = "cloud"
STORAGE_CLOUD_EFFICIENCY = "cloud_efficiency"
STORAGE_CLOUD_SSD = "cloud_ssd"
STORAGE_EPHEMERAL_SSD = "ephemeral_ssd"
STORAGE_ENABLED = "enabled"
STORAGE_DISABLED = "disabled"
pkg/compute/models/usage.go
+2 -1
View File
@@ -2,8 +2,9 @@ package models
import (
"github.com/yunionio/log"
"github.com/yunionio/onecloud/pkg/cloudcommon/db"
"github.com/yunionio/sqlchemy"
"github.com/yunionio/onecloud/pkg/cloudcommon/db"
)
func AttachUsageQuery(
pkg/compute/models/zones.go
+26 -1
View File
@@ -2,14 +2,15 @@ package models
import (
"context"
"database/sql"
"fmt"
"github.com/yunionio/jsonutils"
"github.com/yunionio/log"
"github.com/yunionio/onecloud/pkg/mcclient"
"github.com/yunionio/onecloud/pkg/cloudcommon/db"
"github.com/yunionio/onecloud/pkg/cloudprovider"
"github.com/yunionio/onecloud/pkg/httperrors"
"github.com/yunionio/onecloud/pkg/mcclient"
"github.com/yunionio/pkg/tristate"
"github.com/yunionio/pkg/util/compare"
"github.com/yunionio/sqlchemy"
@@ -288,6 +289,10 @@ return ret
func zoneExtra(zone *SZone, extra *jsonutils.JSONDict) *jsonutils.JSONDict {
usage := zone.GeneralUsage()
extra.Update(jsonutils.Marshal(usage))
region := zone.GetRegion()
if region != nil {
extra.Add(jsonutils.NewString(region.Name), "cloudregion")
}
return extra
}
@@ -607,3 +612,23 @@ func (self *SZone) isUsable() bool {
return false
}
}
func (manager *SZoneManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
regionStr := jsonutils.GetAnyString(query, []string{"region", "region_id", "cloudregion", "cloudregion_id"})
var regionId string
if len(regionStr) > 0 {
regionObj, err := CloudregionManager.FetchByIdOrName("", regionStr)
if err != nil {
if err == sql.ErrNoRows {
return nil, httperrors.NewResourceNotFoundError("Region %s not found", regionStr)
} else {
return nil, httperrors.NewInternalServerError("Query region %s fail %s", regionStr, err)
}
}
regionId = regionObj.GetId()
} else {
regionId = "default"
}
data.Add(jsonutils.NewString(regionId), "cloudregion_id")
return manager.SStatusStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data)
}
pkg/compute/tasks/disk_base_task.go
+14
View File
@@ -3,6 +3,7 @@ package tasks
import (
"context"
"github.com/yunionio/log"
"github.com/yunionio/onecloud/pkg/cloudcommon/db/taskman"
"github.com/yunionio/onecloud/pkg/compute/models"
)
@@ -31,3 +32,16 @@ func (self *SDiskBaseTask) finalReleasePendingUsage(ctx context.Context) {
}
}
}
func (self *SDiskBaseTask) CleanHostSchedCache(disk *models.SDisk) {
storage := disk.GetStorage()
if hosts := storage.GetAllAttachingHosts(); hosts == nil {
log.Errorf("get attaching host error")
} else {
for _, h := range hosts {
if err := h.ClearSchedDescCache(); err != nil {
log.Errorf("host CleanHostSchedCache error: %v", err)
}
}
}
}
pkg/compute/tasks/disk_create_task.go
+68
View File
@@ -0,0 +1,68 @@
package tasks
import (
"context"
"github.com/yunionio/jsonutils"
"github.com/yunionio/onecloud/pkg/cloudcommon/db"
"github.com/yunionio/onecloud/pkg/cloudcommon/db/taskman"
"github.com/yunionio/onecloud/pkg/compute/models"
)
type DiskCreateTask struct {
SDiskBaseTask
}
func init() {
taskman.RegisterTask(DiskCreateTask{})
}
func (self *DiskCreateTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
disk := obj.(*models.SDisk)
storagecache := disk.GetStorage().GetStoragecache()
imageId := disk.GetTemplateId()
if len(imageId) > 0 {
self.SetStage("on_storage_cache_image_complete", nil)
storagecache.StartImageCacheTask(ctx, self.UserCred, imageId, false, self.GetTaskId())
} else {
self.OnStorageCacheImageComplete(ctx, disk)
}
}
func (self *DiskCreateTask) OnStorageCacheImageComplete(ctx context.Context, disk *models.SDisk) {
rebuild, _ := self.GetParams().Bool("rebuild")
snapshot, _ := self.GetParams().GetString("snapshot")
if rebuild {
db.OpsLog.LogEvent(disk, db.ACT_DELOCATE, disk.GetShortDesc(), self.GetUserCred())
}
storage := disk.GetStorage()
host := storage.GetMasterHost()
db.OpsLog.LogEvent(disk, db.ACT_ALLOCATE, disk.GetShortDesc(), self.GetUserCred())
disk.SetStatus(self.GetUserCred(), models.DISK_STARTALLOC, "")
self.SetStage("on_disk_ready", nil)
if err := disk.StartAllocate(host, storage, self.GetTaskId(), self.GetUserCred(), rebuild, snapshot, self); err != nil {
self.OnStartAllocateFailed(ctx, disk, err)
}
}
func (self *DiskCreateTask) OnStartAllocateFailed(ctx context.Context, disk *models.SDisk, resion error) {
disk.SetStatus(self.UserCred, models.DISK_ALLOC_FAILED, resion.Error())
self.SetStageFailed(ctx, resion.Error())
}
func (self *DiskCreateTask) OnDiskReady(ctx context.Context, disk *models.SDisk, data jsonutils.JSONObject) {
diskSize, _ := data.Int("disk_size")
disk.DiskSize = int(diskSize)
disk.DiskFormat, _ = data.GetString("disk_format")
disk.AccessPath, _ = data.GetString("disk_path")
disk.SetStatus(self.UserCred, models.DISK_READY, "")
self.CleanHostSchedCache(disk)
db.OpsLog.LogEvent(disk, db.ACT_ALLOCATE, disk.GetShortDesc(), self.UserCred)
self.SetStageComplete(ctx, nil)
}
func (self *DiskCreateTask) OnDiskReadyFailed(ctx context.Context, disk *models.SDisk, resion error) {
disk.SetStatus(self.UserCred, models.DISK_ALLOC_FAILED, resion.Error())
self.SetStageFailed(ctx, resion.Error())
}
pkg/compute/tasks/disk_delete_task.go
+11 -2
View File
@@ -50,7 +50,10 @@ func (self *DiskDeleteTask) startDeleteDisk(ctx context.Context, disk *models.SD
if isPurge {
self.OnGuestDiskDeleteSucc(ctx, disk, nil)
} else {
// TODO
self.SetStage("on_guest_disk_delete_succ", nil)
if err := host.GetHostDriver().RequestDeallocateDiskOnHost(host, storage, disk, self); err != nil {
self.OnGuestDiskDeleteFailed(ctx, disk, err)
}
}
}
@@ -65,8 +68,14 @@ func (self *DiskDeleteTask) OnGuestDiskDeleteSucc(ctx context.Context, obj db.IS
return
}
disk := obj.(*models.SDisk)
// self.clean_host_sched_cache(disk)
self.CleanHostSchedCache(disk)
db.OpsLog.LogEvent(disk, db.ACT_DELOCATE, disk.GetShortDesc(), self.UserCred)
disk.RealDelete(ctx, self.UserCred)
self.SetStageComplete(ctx, nil)
}
func (self *DiskDeleteTask) OnGuestDiskDeleteFailed(ctx context.Context, disk *models.SDisk, resion error) {
disk.SetStatus(self.GetUserCred(), models.DISK_DEALLOC_FAILED, resion.Error())
self.SetStageFailed(ctx, resion.Error())
db.OpsLog.LogEvent(disk, db.ACT_DELOCATE_FAIL, disk.GetShortDesc(), self.GetUserCred())
}
pkg/compute/tasks/disk_resize_task.go
+82
View File
@@ -0,0 +1,82 @@
package tasks
import (
"context"
"github.com/yunionio/jsonutils"
"github.com/yunionio/log"
"github.com/yunionio/onecloud/pkg/cloudcommon/db"
"github.com/yunionio/onecloud/pkg/cloudcommon/db/taskman"
"github.com/yunionio/onecloud/pkg/compute/models"
)
type DiskResizeTask struct {
SDiskBaseTask
}
func init() {
taskman.RegisterTask(DiskResizeTask{})
}
func (self *DiskResizeTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
disk := obj.(*models.SDisk)
storage := disk.GetStorage()
host := storage.GetMasterHost()
online := disk.GetRuningGuestCount() > 0
if online {
for _, guest := range disk.GetGuests() {
host = guest.GetHost()
}
}
resion := "Cannot find host for disk"
if host == nil || host.HostStatus != models.HOST_ONLINE {
disk.SetStatus(self.GetUserCred(), models.DISK_READY, resion)
self.SetStageFailed(ctx, resion)
db.OpsLog.LogEvent(disk, db.ACT_RESIZE_FAIL, resion, self.GetUserCred())
} else {
disk.SetStatus(self.GetUserCred(), models.DISK_START_RESIZE, "")
for _, guest := range disk.GetGuests() {
guest.SetStatus(self.GetUserCred(), models.VM_RESIZE_DISK, "")
}
self.StartResizeDisk(ctx, host, storage, disk, online)
}
}
func (self *DiskResizeTask) StartResizeDisk(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, online bool) {
log.Infof("Resizing disk on host %s ...", host.GetName())
self.SetStage("on_disk_resize_complete", nil)
size, _ := self.GetParams().Int("size")
proc := host.GetHostDriver().RequestResizeDiskOnHost
if online {
proc = host.GetHostDriver().RequestResizeDiskOnHostOnline
}
if err := proc(host, storage, disk, size, self); err != nil {
log.Errorf("request_resize_disk_on_host: %v", err)
self.OnStartResizeDiskFailed(ctx, err)
}
self.OnStartResizeDiskSucc(ctx, disk)
}
func (self *DiskResizeTask) OnStartResizeDiskSucc(ctx context.Context, disk *models.SDisk) {
disk.SetStatus(self.GetUserCred(), models.DISK_RESIZING, "")
}
func (self *DiskResizeTask) OnStartResizeDiskFailed(ctx context.Context, resion error) {
disk := self.getDisk()
disk.SetStatus(self.GetUserCred(), models.DISK_READY, resion.Error())
self.SetStageFailed(ctx, resion.Error())
db.OpsLog.LogEvent(disk, db.ACT_RESIZE_FAIL, resion.Error(), self.GetUserCred())
}
func (self *DiskResizeTask) OnDiskResizeComplete(ctx context.Context, disk *models.SDisk, data jsonutils.JSONObject) {
disk.SetStatus(self.UserCred, models.DISK_READY, "")
self.CleanHostSchedCache(disk)
db.OpsLog.LogEvent(disk, db.ACT_RESIZE, disk.GetShortDesc(), self.UserCred)
self.SetStageComplete(ctx, disk.GetShortDesc())
self.finalReleasePendingUsage(ctx)
}
func (self *DiskResizeTask) OnDiskResizeCompleteFailed(ctx context.Context, disk *models.SDisk, resion error) {
disk.SetStatus(self.UserCred, models.DISK_READY, resion.Error())
db.OpsLog.LogEvent(disk, db.ACT_RESIZE_FAIL, disk.GetShortDesc(), self.UserCred)
}
pkg/compute/tasks/guest_sync_task.go
+68
View File
@@ -0,0 +1,68 @@
package tasks
import (
"context"
"github.com/yunionio/jsonutils"
"github.com/yunionio/log"
"github.com/yunionio/onecloud/pkg/cloudcommon/db"
"github.com/yunionio/onecloud/pkg/cloudcommon/db/taskman"
"github.com/yunionio/onecloud/pkg/compute/models"
)
type GuestSyncConfTask struct {
SGuestBaseTask
}
func init() {
taskman.RegisterTask(GuestSyncConfTask{})
}
func (self *GuestSyncConfTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
guest := obj.(*models.SGuest)
db.OpsLog.LogEvent(guest, db.ACT_SYNC_CONF, nil, self.UserCred)
if host := guest.GetHost(); host == nil {
self.SetStageFailed(ctx, "No host for sync")
return
} else {
self.SetStage("on_sync_complete", nil)
if err := guest.GetDriver().RequestSyncConfigOnHost(ctx, guest, host, self); err != nil {
self.SetStageFailed(ctx, err.Error())
log.Errorf("SyncConfTask faled %v", err)
}
}
}
func (self *GuestSyncConfTask) OnSyncComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
guest := obj.(*models.SGuest)
if fw_only, _ := self.GetParams().Bool("fw_only"); fw_only {
db.OpsLog.LogEvent(guest, db.ACT_SYNC_CONF, nil, self.UserCred)
self.OnSyncComplete(ctx, obj, guest.GetShortDesc())
} else if data.Contains("task") {
self.SetStage("on_disk_sync_complete", nil)
} else {
self.OnDiskSyncComplete(ctx, guest, data)
}
}
func (self *GuestSyncConfTask) OnDiskSyncComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
self.SetStage("on_sync_status_complete", nil)
guest.StartSyncstatus(ctx, self.GetUserCred(), self.GetTaskId())
}
func (self *GuestSyncConfTask) OnDiskSyncCompleteFailed(ctx context.Context, obj db.IStandaloneModel, resion error) {
guest := obj.(*models.SGuest)
db.OpsLog.LogEvent(guest, db.ACT_SYNC_CONF_FAIL, resion.Error(), self.UserCred)
log.Errorf("Guest sync config failed: %v", resion)
}
func (self *GuestSyncConfTask) OnSyncCompleteFailed(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
guest := obj.(*models.SGuest)
guest.SetStatus(self.GetUserCred(), models.VM_SYNC_FAIL, data.String())
log.Errorf("Guest sync config failed: %v", data.String())
db.OpsLog.LogEvent(guest, db.ACT_SYNC_CONF_FAIL, data.String(), self.UserCred)
}
func (self *GuestSyncConfTask) OnSyncStatusComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
self.SetStageComplete(ctx, nil)
}
pkg/dns/dns.go
+414
View File
@@ -0,0 +1,414 @@
package dns
import (
"context"
"database/sql"
"errors"
"fmt"
"strconv"
"strings"
"github.com/coredns/coredns/plugin"
"github.com/coredns/coredns/plugin/etcd/msg"
"github.com/coredns/coredns/plugin/pkg/dnsutil"
"github.com/coredns/coredns/plugin/pkg/fall"
"github.com/coredns/coredns/plugin/pkg/upstream"
"github.com/coredns/coredns/request"
_ "github.com/go-sql-driver/mysql"
"github.com/mholt/caddy"
"github.com/miekg/dns"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/client-go/kubernetes"
ylog "github.com/yunionio/log"
"github.com/yunionio/pkg/utils"
"github.com/yunionio/sqlchemy"
"github.com/yunionio/onecloud/pkg/cloudcommon/db"
"github.com/yunionio/onecloud/pkg/compute/models"
"github.com/yunionio/onecloud/pkg/util/k8s"
)
const (
PluginName string = "yunion"
// defaultTTL to apply to all answers
defaultTTL = 10
)
var (
DNSTypeMap map[uint16]string = map[uint16]string{
dns.TypeA: "A",
dns.TypeAAAA: "AAAA",
dns.TypeTXT: "TXT",
dns.TypeCNAME: "CNAME",
dns.TypePTR: "PTR",
dns.TypeMX: "MX",
dns.TypeSRV: "SRV",
dns.TypeSOA: "SOA",
dns.TypeNS: "NS",
}
)
type SRegionDNS struct {
Next plugin.Handler
Fall fall.F
Zones []string
PrimaryZone string
Upstream upstream.Upstream
SqlConnection string
K8sConfigFile string
K8sClient *kubernetes.Clientset
}
func New() *SRegionDNS {
r := new(SRegionDNS)
return r
}
func (r *SRegionDNS) initDB(c *caddy.Controller) error {
dialect, sqlStr, err := utils.TransSQLAchemyURL(r.SqlConnection)
if err != nil {
return err
}
dbConn, err := sql.Open(dialect, sqlStr)
if err != nil {
return err
}
sqlchemy.SetDB(dbConn)
db.InitAllManagers()
c.OnShutdown(func() error {
r.CloseDB()
return nil
})
return nil
}
func (r *SRegionDNS) initK8s(c *caddy.Controller) {
cli, err := k8s.NewClientByFile(r.K8sConfigFile, nil)
if err != nil {
ylog.Errorf("Init kubernetes client error: %v", err)
return
}
r.K8sClient = cli
pods, err := cli.CoreV1().Pods("").List(metav1.ListOptions{})
if err != nil {
ylog.Errorf("Get all pods in kubernetes cluster error: %v", err)
return
}
ylog.Infof("Init k8s client success, %d pods in the cluster", len(pods.Items))
}
func (r *SRegionDNS) CloseDB() {
sqlchemy.CloseDB()
}
func (r *SRegionDNS) ServeDNS(ctx context.Context, w dns.ResponseWriter, rmsg *dns.Msg) (int, error) {
var (
records []dns.RR
extra []dns.RR
err error
)
opt := plugin.Options{}
state := request.Request{W: w, Req: rmsg, Context: ctx}
zone := plugin.Zones(r.Zones).Matches(state.Name())
switch state.QType() {
case dns.TypeA:
ylog.Debugf("A question: %#v", state)
records, err = plugin.A(r, zone, state, nil, opt)
case dns.TypeAAAA:
ylog.Debugf("AAAA question: %#v", state)
records, err = plugin.AAAA(r, zone, state, nil, opt)
case dns.TypeTXT:
ylog.Debugf("TXT question: %#v", state)
records, err = plugin.TXT(r, zone, state, opt)
case dns.TypeCNAME:
ylog.Debugf("CNAME question: %#v", state)
records, err = plugin.CNAME(r, zone, state, opt)
case dns.TypePTR:
ylog.Debugf("PTR question: %#v", state)
records, err = plugin.PTR(r, zone, state, opt)
case dns.TypeMX:
ylog.Debugf("MX question: %#v", state)
records, extra, err = plugin.MX(r, zone, state, opt)
case dns.TypeSRV:
ylog.Debugf("SRV question: %#v", state)
records, extra, err = plugin.SRV(r, zone, state, opt)
case dns.TypeSOA:
ylog.Debugf("SOA question: %#v", state)
records, err = plugin.SOA(r, zone, state, opt)
case dns.TypeNS:
ylog.Debugf("NS question: %#v", state)
if state.Name() == zone {
records, extra, err = plugin.NS(r, zone, state, opt)
break
}
fallthrough
default:
ylog.Warningf("Not processed state: %#v", state)
// Do a fake A lookup, so we can distinguish between NODATA and NXDOMAIN
_, err = plugin.A(r, zone, state, nil, opt)
}
if r.IsNameError(err) {
if r.Fall.Through(state.Name()) {
return plugin.NextOrFailure(r.Name(), r.Next, ctx, w, rmsg)
}
return plugin.BackendError(r, zone, dns.RcodeNameError, state, nil /* err */, opt)
}
if err != nil {
return plugin.BackendError(r, zone, dns.RcodeServerFailure, state, err, opt)
}
if len(records) == 0 {
return plugin.BackendError(r, zone, dns.RcodeSuccess, state, err, opt)
}
m := new(dns.Msg)
m.SetReply(rmsg)
m.Authoritative, m.RecursionAvailable = true, true
m.Answer = append(m.Answer, records...)
m.Extra = append(m.Extra, extra...)
state.SizeAndDo(m)
m, _ = state.Scrub(m)
w.WriteMsg(m)
return dns.RcodeSuccess, nil
}
var (
errNoItems = errors.New("no items found")
)
// Services implements the ServiceBackend interface
func (r *SRegionDNS) Services(state request.Request, exact bool, opt plugin.Options) (services []msg.Service, err error) {
switch state.QType() {
case dns.TypeTXT:
t, _ := dnsutil.TrimZone(state.Name(), state.Zone)
segs := dns.SplitDomainName(t)
if len(segs) != 1 {
return nil, fmt.Errorf("yunion region: TXT query can onlyu be for dns-version: %s", state.QName())
}
if segs[0] != "dns-version" {
return nil, nil
}
svc := msg.Service{Text: "0.0.1", TTL: 28800, Key: msg.Path(state.QName(), "coredns")}
return []msg.Service{svc}, nil
case dns.TypeNS:
ns := r.nsAddr()
svc := msg.Service{Host: ns.A.String(), Key: msg.Path(state.QName(), "coredns")}
return []msg.Service{svc}, nil
}
if state.QType() == dns.TypeA && isDefaultNS(state.Name(), state.Zone) {
// If this is an A request for "ns.dns", respond with a "fake" record for coredns.
// SOA records always use this hardcoded name
ns := r.nsAddr()
svc := msg.Service{Host: ns.A.String(), Key: msg.Path(state.QName(), "coredns")}
return []msg.Service{svc}, nil
}
services, err = r.Records(state, false)
ylog.Debugf("Get records: %#v, error: %v", services, err)
return
}
// Lookup implements the ServiceBackend interface
func (r *SRegionDNS) Lookup(state request.Request, name string, typ uint16) (*dns.Msg, error) {
return r.Upstream.Lookup(state, name, typ)
}
// IsNameError implements the ServiceBackend interface
func (r *SRegionDNS) IsNameError(err error) bool {
return err == errNoItems
}
// Records looks up records in region mysql
func (r *SRegionDNS) Records(state request.Request, exact bool) ([]msg.Service, error) {
req, e := parseRequest(state)
if e != nil {
return nil, e
}
return r.findRecords(req)
}
func (r *SRegionDNS) getHostIpWithName(req *recordRequest) []string {
name := req.QueryName()
host, _ := models.HostManager.FetchByName("", name)
if host == nil {
return nil
}
ip := host.(*models.SHost).AccessIp
if len(ip) == 0 {
return nil
}
return []string{ip}
}
func (r *SRegionDNS) getGuestIpWithName(req *recordRequest) []string {
ips := []string{}
name := req.QueryName()
projectId := req.ProjectId()
isExitOnly := req.IsExitOnly()
ips = models.GuestManager.GetIpInProjectWithName(projectId, name, isExitOnly)
return ips
}
func (r *SRegionDNS) getK8sServiceBackends(req *recordRequest) ([]string, error) {
queryInfo := req.GetK8sQueryInfo()
pods, err := r.getK8sServicePods(queryInfo.Namespace, queryInfo.ServiceName)
if err != nil {
return nil, err
}
ips := make([]string, 0)
for _, pod := range pods {
ip := pod.Status.PodIP
if len(ip) != 0 {
ips = append(ips, ip)
}
}
return ips, nil
}
func (r *SRegionDNS) getK8sServicePods(namespace, name string) ([]v1.Pod, error) {
cli := r.K8sClient
svc, err := cli.CoreV1().Services(namespace).Get(name, metav1.GetOptions{})
if err != nil {
return nil, err
}
labelSelector := labels.SelectorFromSet(svc.Spec.Selector)
pods, err := cli.CoreV1().Pods(namespace).List(metav1.ListOptions{
LabelSelector: labelSelector.String(),
FieldSelector: fields.Everything().String(),
})
if err != nil {
return nil, err
}
return pods.Items, nil
}
func (r *SRegionDNS) Name() string {
return PluginName
}
func (r *SRegionDNS) queryLocalDnsRecords(req *recordRequest) (recs []msg.Service, err error) {
ips := models.DnsRecordManager.QueryDnsIps(req.ProjectId(), req.Name(), req.Type())
if len(ips) == 0 {
err = errNoItems
return
}
for _, ip := range ips {
var s = msg.Service{}
var ttl uint32 = uint32(ip.Ttl)
if ttl == 0 {
ttl = defaultTTL
}
if req.IsSRV() {
parts := strings.SplitN(ip.Addr, ":", 2)
if len(parts) != 2 {
err = fmt.Errorf("Invalid SRV records: %q", ip.Addr)
return
}
port, e := strconv.Atoi(parts[1])
if e != nil {
err = e
return
}
s = msg.Service{Host: parts[0], Port: port, TTL: ttl}
} else {
s = msg.Service{Host: ip.Addr, TTL: ttl}
}
recs = append(recs, s)
}
return
}
func (r *SRegionDNS) IsCloudNetworkIp(req *recordRequest) bool {
if req.network != nil {
return true
}
return false
}
func (r *SRegionDNS) IsK8sClientReady() bool {
return r.K8sClient != nil
}
func (r *SRegionDNS) isMyDomain(req *recordRequest) bool {
zones := []string{fmt.Sprintf("%s.", r.PrimaryZone)}
zone := plugin.Zones(zones).Matches(req.state.Name())
if zone != "" {
return true
}
return false
}
func (r *SRegionDNS) findRecords(req *recordRequest) (recs []msg.Service, err error) {
// 1. try local dns records table
recs, err = r.queryLocalDnsRecords(req)
if len(recs) != 0 {
return
}
isMyDomain := r.isMyDomain(req)
isCloudIp := r.IsCloudNetworkIp(req)
// 2. not my domain and src ip not in cloud network table
// query from upstream
if !isMyDomain && !isCloudIp {
err = errNoItems
return
}
// 3. internal query
ips, err := r.findInternalRecordIps(req)
return ips2DnsRecords(ips), err
}
func (r *SRegionDNS) findInternalRecordIps(req *recordRequest) ([]string, error) {
// 1. try host table
ip := r.getHostIpWithName(req)
if len(ip) != 0 {
return ip, nil
}
// 2. try guest table
ip = r.getGuestIpWithName(req)
if len(ip) != 0 {
return ip, nil
}
if !r.IsK8sClientReady() {
ylog.Warningf("K8s client not ready, skip it.")
return nil, errNoItems
}
// 3. try k8s service backends
ips, err := r.getK8sServiceBackends(req)
if len(ips) != 0 {
return ips, nil
}
if err != nil {
ylog.Errorf("Get k8s service backends error: %v", err)
}
return nil, errNoItems
}
func ips2DnsRecords(ips []string) []msg.Service {
recs := make([]msg.Service, 0)
for _, ip := range ips {
s := msg.Service{Host: ip, TTL: defaultTTL}
recs = append(recs, s)
}
return recs
}
pkg/dns/ns.go
+19
View File
@@ -0,0 +1,19 @@
package dns
import (
//"net"
"strings"
"github.com/miekg/dns"
)
const defaultNSName = "ns.dns."
func isDefaultNS(name, zone string) bool {
return strings.Index(name, defaultNSName) == 0 && strings.Index(name, zone) == len(defaultNSName)
}
func (r *SRegionDNS) nsAddr() *dns.A {
//rr := new(dns.A)
return nil
}
pkg/dns/parse.go
+102
View File
@@ -0,0 +1,102 @@
package dns
import (
"strings"
"github.com/coredns/coredns/plugin/pkg/dnsutil"
"github.com/coredns/coredns/request"
"github.com/miekg/dns"
"github.com/yunionio/pkg/tristate"
"github.com/yunionio/onecloud/pkg/compute/models"
)
type recordRequest struct {
state request.Request
domainSegs []string
guest *models.SGuest
host *models.SHost
network *models.SNetwork
}
func parseRequest(state request.Request) (r *recordRequest, err error) {
base, _ := dnsutil.TrimZone(state.Name(), state.Zone)
segs := dns.SplitDomainName(base)
r = &recordRequest{
state: state,
domainSegs: segs,
}
srcIP := r.SrcIP4()
r.guest = models.GuestnetworkManager.GetGuestByAddress(srcIP)
r.host = models.HostnetworkManager.GetHostByAddress(srcIP)
r.network, _ = models.NetworkManager.GetNetworkOfIP(srcIP, "", tristate.None)
return
}
func (r recordRequest) Name() string {
//fullName, _ := dnsutil.TrimZone(r.state.Name(), "")
name := r.state.Name()
name = strings.TrimSuffix(name, ".")
return name
}
func (r recordRequest) QueryName() string {
seps := strings.Split(r.Name(), ".")
if len(seps) == 0 {
return ""
}
return seps[0]
}
func (r recordRequest) Type() string {
return DNSTypeMap[r.state.QType()]
}
func (r recordRequest) IsSRV() bool {
return r.Type() == DNSTypeMap[dns.TypeSRV]
}
func (r recordRequest) SrcIP4() string {
ip := r.state.IP()
return ip
}
func (r recordRequest) ProjectId() string {
if r.guest != nil {
return r.guest.ProjectId
}
if r.network != nil {
return r.network.ProjectId
}
return ""
}
func (r recordRequest) IsExitOnly() bool {
if r.guest == nil {
return false
}
return r.guest.IsExitOnly()
}
type K8sQueryInfo struct {
ServiceName string
Namespace string
}
func (r recordRequest) GetK8sQueryInfo() K8sQueryInfo {
parts := strings.SplitN(r.Name(), ".", 3)
var svcName string
var namespace string
if len(parts) >= 2 {
svcName = parts[0]
namespace = parts[1]
} else {
svcName = parts[0]
namespace = "default"
}
return K8sQueryInfo{
ServiceName: svcName,
Namespace: namespace,
}
}
pkg/dns/reverse.go
+61
View File
@@ -0,0 +1,61 @@
package dns
import (
"strings"
"github.com/coredns/coredns/plugin"
"github.com/coredns/coredns/plugin/etcd/msg"
"github.com/coredns/coredns/plugin/pkg/dnsutil"
"github.com/coredns/coredns/request"
"github.com/yunionio/log"
"github.com/yunionio/onecloud/pkg/compute/models"
)
// Reverse implements the ServiceBackend interface
func (r *SRegionDNS) Reverse(state request.Request, exact bool, opt plugin.Options) (services []msg.Service, err error) {
ip := dnsutil.ExtractAddressFromReverse(state.Name())
if ip == "" {
_, e := r.Records(state, exact)
return nil, e
}
records, err := r.getNameForIp(ip, state)
if err != nil {
log.Errorf("Reverse get name for ip: %v", err)
}
if len(records) == 0 {
return records, errNoItems
}
return records, nil
}
func (r *SRegionDNS) getNameForIp(ip string, state request.Request) ([]msg.Service, error) {
req, e := parseRequest(state)
if e != nil {
return nil, e
}
// 1. try local dns records table
records := models.DnsRecordManager.QueryDnsIps(req.ProjectId(), req.Name(), req.Type())
for _, rec := range records {
return []msg.Service{{Host: rec.Addr, TTL: uint32(rec.Ttl)}}, nil
}
// 2. try hosts table
host := models.HostnetworkManager.GetHostByAddress(ip)
if host != nil {
return []msg.Service{{Host: r.joinDomain(host.Name), TTL: defaultTTL}}, nil
}
// 3. try guests table
guest := models.GuestnetworkManager.GetGuestByAddress(ip)
if guest != nil {
return []msg.Service{{Host: r.joinDomain(guest.Name), TTL: defaultTTL}}, nil
}
return nil, errNoItems
}
func (r *SRegionDNS) joinDomain(name string) string {
return strings.Join([]string{name, r.PrimaryZone}, ".")
}
pkg/dns/setup.go
+108
View File
@@ -0,0 +1,108 @@
package dns
import (
"fmt"
"os"
"github.com/coredns/coredns/core/dnsserver"
"github.com/coredns/coredns/plugin"
"github.com/coredns/coredns/plugin/pkg/upstream"
"github.com/mholt/caddy"
"github.com/yunionio/pkg/util/regutils"
)
func init() {
caddy.RegisterPlugin(PluginName, caddy.Plugin{
ServerType: "dns",
Action: setup,
})
}
func setup(c *caddy.Controller) error {
os.Stderr = os.Stdout
rDNS, err := regionDNSParse(c)
if err != nil {
return plugin.Error(PluginName, err)
}
if rDNS.PrimaryZone == "" {
return fmt.Errorf("dns_domain must provided")
}
if !regutils.MatchDomainName(rDNS.PrimaryZone) {
return fmt.Errorf("dns_domain %q not match domain format", rDNS.PrimaryZone)
}
err = rDNS.initDB(c)
if err != nil {
return plugin.Error(PluginName, err)
}
rDNS.initK8s(c)
dnsserver.GetConfig(c).AddPlugin(func(next plugin.Handler) plugin.Handler {
rDNS.Next = next
return rDNS
})
return nil
}
func regionDNSParse(c *caddy.Controller) (*SRegionDNS, error) {
return parseConfig(c)
}
func parseConfig(c *caddy.Controller) (*SRegionDNS, error) {
rDNS := New()
for c.Next() {
rDNS.Zones = c.RemainingArgs()
if len(rDNS.Zones) == 0 {
rDNS.Zones = make([]string, len(c.ServerBlockKeys))
copy(rDNS.Zones, c.ServerBlockKeys)
}
for i, str := range rDNS.Zones {
rDNS.Zones[i] = plugin.Host(str).Normalize()
}
if c.NextBlock() {
for {
switch c.Val() {
case "dns_domain":
if !c.NextArg() {
return nil, c.ArgErr()
}
rDNS.PrimaryZone = c.Val()
case "fallthrough":
rDNS.Fall.SetZonesFromArgs(c.RemainingArgs())
case "sql_connection":
if !c.NextArg() {
return nil, c.ArgErr()
}
rDNS.SqlConnection = c.Val()
case "kube_config":
if !c.NextArg() {
return nil, c.ArgErr()
}
rDNS.K8sConfigFile = c.Val()
case "upstream":
args := c.RemainingArgs()
u, err := upstream.New(args)
if err != nil {
return nil, err
}
rDNS.Upstream = u
default:
if c.Val() != "}" {
return nil, c.Errf("unknown property %q", c.Val())
}
}
if !c.Next() {
break
}
}
}
}
return rDNS, nil
}
pkg/dns/xfr.go
+24
View File
@@ -0,0 +1,24 @@
package dns
import (
"context"
"time"
"github.com/coredns/coredns/request"
"github.com/miekg/dns"
)
// Serial implements the Transferer interface
func (r *SRegionDNS) Serial(state request.Request) uint32 {
return uint32(time.Now().Unix())
}
// MinTTL implements the Transferer interface
func (r *SRegionDNS) MinTTL(state request.Request) uint32 {
return 30
}
// Transferer implements the Transferer interface
func (r *SRegionDNS) Transfer(ctx context.Context, state request.Request) (int, error) {
return dns.RcodeServerFailure, nil
}
pkg/httperrors/errors.go
+7
View File
@@ -39,6 +39,13 @@ func NewResourceNotReadyError(msg string, params ...interface{}) *httputils.JSON
return NewJsonClientError(500, "ResourceNotReadyError", msg)
}
func NewOutOfResourceError(msg string, params ...interface{}) *httputils.JSONClientError {
if len(params) > 0 {
msg = fmt.Sprintf(msg, params...)
}
return NewJsonClientError(500, "NewOutOfResourceError", msg)
}
func NewServerStatusError(msg string, params ...interface{}) *httputils.JSONClientError {
if len(params) > 0 {
msg = fmt.Sprintf(msg, params...)
pkg/scheduler/algorithm/predicates/guest/hypervisor_predicate.go
+14 -2
View File
@@ -2,6 +2,7 @@ package guest
import (
"github.com/yunionio/log"
"github.com/yunionio/onecloud/pkg/scheduler/algorithm/predicates"
"github.com/yunionio/onecloud/pkg/scheduler/api"
"github.com/yunionio/onecloud/pkg/scheduler/core"
@@ -35,6 +36,18 @@ func hostHasContainerTag(c core.Candidater) bool {
return false
}
func hostAllowRunContainer(c core.Candidater) bool {
hostType := c.Get("HostType")
if hostType == api.HostTypeKubelet {
return true
}
if hostHasContainerTag(c) {
log.Debugf("Host %q has %q tag, allow it run container", c.IndexKey(), CONTAINER_ALLOWED_TAG)
return true
}
return false
}
func (f *HypervisorPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) {
h := predicates.NewPredicateHelper(f, u, c)
@@ -42,8 +55,7 @@ func (f *HypervisorPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []
guestNeedType := u.SchedData().Hypervisor
if guestNeedType != hostType {
if guestNeedType == api.SchedTypeContainer && hostHasContainerTag(c) {
log.Debugf("Host %q has %q tag, allow it run container", c.IndexKey(), CONTAINER_ALLOWED_TAG)
if guestNeedType == api.SchedTypeContainer && hostAllowRunContainer(c) {
return h.GetResult()
}
h.Exclude2(f.Name(), hostType, guestNeedType)
pkg/scheduler/api/types.go
+1
View File
@@ -17,6 +17,7 @@ const (
SchedTypeKvm = "kvm"
HostHypervisorForKvm = "hypervisor"
HostTypeAliyun = "aliyun"
HostTypeKubelet = "kubelet"
AggregateStrategyRequire = "require"
AggregateStrategyExclude = "exclude"
pkg/util/aliyun/disk.go
+4
View File
@@ -96,6 +96,10 @@ func (self *SDisk) GetId() string {
return self.DiskId
}
func (self *SDisk) Delete() error {
return self.storage.zone.region.deleteDisk(self.DiskId)
}
func (self *SDisk) GetName() string {
return self.DiskId
}
pkg/util/aliyun/instance.go
+24
View File
@@ -10,6 +10,7 @@ import (
"github.com/yunionio/onecloud/pkg/compute/models"
"github.com/yunionio/pkg/util/osprofile"
"github.com/yunionio/pkg/util/seclib"
"github.com/yunionio/pkg/util/secrules"
"github.com/yunionio/pkg/utils"
)
@@ -184,6 +185,10 @@ func (self *SInstance) IsEmulated() bool {
return false
}
func (self *SInstance) getVpc() (*SVpc, error) {
return self.host.zone.region.getVpc(self.VpcAttributes.VpcId)
}
func (self *SInstance) fetchDisks() error {
disks, total, err := self.host.zone.region.GetDisks(self.InstanceId, "", "", nil, 0, 50)
if err != nil {
@@ -501,3 +506,22 @@ func (self *SInstance) GetVNCInfo() (jsonutils.JSONObject, error) {
ret.Add(jsonutils.NewString(self.InstanceId), "instance_id")
return ret, nil
}
func (self *SInstance) SyncSecurityGroup(secgroupId string, rules []*secrules.SecurityRule) error {
if vpc, err := self.getVpc(); err != nil {
return err
} else if len(secgroupId) == 0 {
for _, secgrpId := range self.SecurityGroupIds.SecurityGroupId {
if err := vpc.revokeSecurityGroup(secgrpId, self.InstanceId); err != nil {
return err
}
}
} else if secgrpId, err := vpc.syncSecurityGroup(secgroupId, rules); err != nil {
return err
} else if err := vpc.assignSecurityGroup(secgrpId, self.InstanceId); err != nil {
return err
} else {
self.SecurityGroupIds.SecurityGroupId = append(self.SecurityGroupIds.SecurityGroupId, secgrpId)
}
return nil
}
pkg/util/aliyun/region.go
+1
View File
@@ -291,6 +291,7 @@ func (self *SRegion) getVpc(vpcId string) (*SVpc, error) {
if total != 1 {
return nil, cloudprovider.ErrNotFound
}
vpcs[0].region = self
return &vpcs[0], nil
}
pkg/util/aliyun/securitygroup.go
+200 -4
View File
@@ -2,9 +2,13 @@ package aliyun
import (
"fmt"
"strings"
"time"
"github.com/deckarep/golang-set"
"github.com/yunionio/jsonutils"
"github.com/yunionio/log"
"github.com/yunionio/onecloud/pkg/httperrors"
"github.com/yunionio/pkg/util/secrules"
"github.com/yunionio/pkg/utils"
)
@@ -124,13 +128,16 @@ func (self *SRegion) createSecurityGroup(vpcId string, name string, desc string)
func (self *SRegion) addSecurityGroupRule(secGrpId string, rule *secrules.SecurityRule) error {
params := make(map[string]string)
params["RegionId"] = self.RegionId
params["SecurityGroupId"] = secGrpId
params["NicType"] = string(IntranetNicType)
params["PortRange"] = fmt.Sprintf("%d/%d", rule.PortStart, rule.PortEnd)
protocol := rule.Protocol
if len(rule.Protocol) == 0 {
if len(rule.Protocol) == 0 || rule.Protocol == secrules.PROTO_ANY {
protocol = "all"
params["PortRange"] = "-1/-1"
}
params["IpProtocol"] = protocol
params["PortRange"] = fmt.Sprintf("%d/%d", rule.PortStart, rule.PortEnd)
if rule.Action == secrules.SecurityRuleAllow {
params["Policy"] = "accept"
} else {
@@ -143,7 +150,7 @@ func (self *SRegion) addSecurityGroupRule(secGrpId string, rule *secrules.Securi
} else {
params["SourceCidrIp"] = "0.0.0.0/0"
}
params["DestCidrIp"] = "0.0.0.0/0"
log.Debugf("add Security Group params: %v", params)
_, err := self.ecsRequest("AuthorizeSecurityGroup", params)
return err
} else { // rule.Direction == secrules.SecurityRuleEgress {
@@ -152,12 +159,51 @@ func (self *SRegion) addSecurityGroupRule(secGrpId string, rule *secrules.Securi
} else {
params["DestCidrIp"] = "0.0.0.0/0"
}
params["SourceCidrIp"] = "0.0.0.0/0"
log.Debugf("add Security Group params: %v", params)
_, err := self.ecsRequest("AuthorizeSecurityGroupEgress", params)
return err
}
}
func (self *SRegion) delSecurityGroupRule(secGrpId string, rule *secrules.SecurityRule) error {
params := make(map[string]string)
params["RegionId"] = self.RegionId
params["SecurityGroupId"] = secGrpId
params["NicType"] = string(IntranetNicType)
params["PortRange"] = fmt.Sprintf("%d/%d", rule.PortStart, rule.PortEnd)
protocol := rule.Protocol
if len(rule.Protocol) == 0 || rule.Protocol == secrules.PROTO_ANY {
protocol = "all"
params["PortRange"] = "-1/-1"
}
params["IpProtocol"] = protocol
if rule.Action == secrules.SecurityRuleAllow {
params["Policy"] = "accept"
} else {
params["Policy"] = "drop"
}
params["Priority"] = fmt.Sprintf("%d", rule.Priority)
if rule.Direction == secrules.SecurityRuleIngress {
if rule.IPNet != nil {
params["SourceCidrIp"] = rule.IPNet.String()
} else {
params["SourceCidrIp"] = "0.0.0.0/0"
}
log.Debugf("del Security Group params: %v", params)
_, err := self.ecsRequest("RevokeSecurityGroup", params)
return err
} else { // rule.Direction == secrules.SecurityRuleEgress {
if rule.IPNet != nil {
params["DestCidrIp"] = rule.IPNet.String()
} else {
params["DestCidrIp"] = "0.0.0.0/0"
}
log.Debugf("del Security Group params: %v", params)
_, err := self.ecsRequest("RevokeSecurityGroupEgress", params)
return err
}
}
func (self *SRegion) createDefaultSecurityGroup(vpcId string) (string, error) {
secId, err := self.createSecurityGroup(vpcId, "", "")
if err != nil {
@@ -189,3 +235,153 @@ func (self *SRegion) createDefaultSecurityGroup(vpcId string) (string, error) {
}
return secId, nil
}
func (self *SRegion) getSecurityGroupByTag(vpcId, secgroupId string) (*SSecurityGroup, error) {
params := make(map[string]string)
params["RegionId"] = self.RegionId
if len(vpcId) > 0 {
params["VpcId"] = vpcId
}
params["Tag.1.Key"] = "id"
params["Tag.1.Value"] = secgroupId
secgrps := make([]SSecurityGroup, 0)
if body, err := self.ecsRequest("DescribeSecurityGroups", params); err != nil {
return nil, err
} else if err := body.Unmarshal(&secgrps, "SecurityGroups", "SecurityGroup"); err != nil {
return nil, err
} else if len(secgrps) != 1 {
return nil, httperrors.NewNotFoundError("failed to find SecurityGroup %s", secgroupId)
}
return &secgrps[0], nil
}
func (self *SPermission) toString() string {
action := secrules.SecurityRuleDeny
if strings.ToLower(self.Policy) == "accept" {
action = secrules.SecurityRuleAllow
}
direction := "in"
if self.Direction == "egress" {
direction = "out"
}
cidr := self.SourceCidrIp
if direction == "out" {
cidr = self.DestCidrIp
}
protocol := strings.ToLower(self.IpProtocol)
if protocol == "all" {
protocol = "any"
}
port, ports := "", strings.Split(self.PortRange, "/")
if ports[0] == ports[1] && (ports[0] != "-1") {
port = ports[0]
} else {
port = fmt.Sprintf("%s-%s", ports[0], ports[1])
}
return fmt.Sprintf("%s:%s %s %s %s", direction, string(action), cidr, protocol, port)
}
func (self *SRegion) addTagToSecurityGroup(secgroupId, key, value string, index int) error {
if index > 5 || index < 1 {
index = 1
}
params := map[string]string{"ResourceType": "securitygroup", "ResourceId": secgroupId}
params[fmt.Sprintf("Tag.%d.Key", index)] = key
params[fmt.Sprintf("Tag.%d.Value", index)] = value
_, err := self.ecsRequest("AddTags", params)
return err
}
func (self *SRegion) revokeSecurityGroup(secgroupId, instanceId string) error {
if secgroup, err := self.GetSecurityGroupDetails(secgroupId); err != nil {
return err
} else {
for _, r := range secgroup.Permissions.Permission {
if rule, err := secrules.ParseSecurityRule(r.toString()); err != nil {
return err
} else {
rule.Priority = r.Priority
if err := self.delSecurityGroupRule(secgroup.SecurityGroupId, rule); err != nil {
return err
}
}
}
if rule, err := secrules.ParseSecurityRule("in:allow any"); err != nil {
rule.Priority = 100
if err := self.addSecurityGroupRule(secgroup.SecurityGroupId, rule); err != nil {
return err
}
}
if rule, err := secrules.ParseSecurityRule("out:allow any"); err != nil {
rule.Priority = 100
if err := self.addSecurityGroupRule(secgroup.SecurityGroupId, rule); err != nil {
return err
}
}
}
return nil
}
func (self *SRegion) syncSecgroupRules(secgroupId string, rules []*secrules.SecurityRule) error {
if secgroup, err := self.GetSecurityGroupDetails(secgroupId); err != nil {
return err
} else {
newRules, newStr := make(map[string]*secrules.SecurityRule), mapset.NewSet()
for _, rule := range rules {
rule.Priority = 101 - rule.Priority
if len(rule.Ports) > 0 {
for _, port := range rule.Ports {
rule.PortStart, rule.PortEnd = port, port
if jsonStr := jsonutils.Marshal(rule).String(); newStr.Add(jsonStr) {
newRules[jsonStr] = rule
}
}
continue
} else if rule.PortStart == 0 || rule.PortEnd == 0 {
rule.PortStart, rule.PortEnd = -1, -1
}
if jsonStr := jsonutils.Marshal(rule).String(); newStr.Add(jsonStr) {
newRules[jsonStr] = rule
}
}
oldRules, oldStr := make(map[string]*secrules.SecurityRule), mapset.NewSet()
for _, r := range secgroup.Permissions.Permission {
if rule, err := secrules.ParseSecurityRule(r.toString()); err != nil {
return err
} else {
rule.Priority = r.Priority
if jsonStr := jsonutils.Marshal(rule).String(); oldStr.Add(jsonStr) {
oldRules[jsonStr] = rule
}
}
}
newStr.Each(func(r interface{}) bool {
rStr := r.(string)
if !oldStr.Contains(r) {
rule := newRules[rStr]
log.Debugf("add secgroup rule: %v Priority: %d", rule, rule.Priority)
self.addSecurityGroupRule(secgroupId, rule)
}
return false
})
oldStr.Each(func(r interface{}) bool {
rStr := r.(string)
if !newStr.Contains(r) {
rule := oldRules[rStr]
log.Debugf("del secgroup rule: %v Priority: %d", rule, rule.Priority)
self.delSecurityGroupRule(secgroupId, rule)
}
return false
})
}
return nil
}
func (self *SRegion) assignSecurityGroup(secgroupId, instanceId string) error {
params := map[string]string{"InstanceId": instanceId, "SecurityGroupId": secgroupId}
_, err := self.ecsRequest("JoinSecurityGroup", params)
return err
}
pkg/util/aliyun/storage.go
+9
View File
@@ -113,3 +113,12 @@ func (self *SStorage) CreateIDisk(name string, sizeGb int, desc string) (cloudpr
disk.storage = self
return disk, nil
}
func (self *SStorage) GetIDisk(idStr string) (cloudprovider.ICloudDisk, error) {
if disk, err := self.zone.region.getDisk(idStr); err != nil {
return nil, err
} else {
disk.storage = self
return disk, nil
}
}
pkg/util/aliyun/vpc.go
+35
View File
@@ -4,8 +4,10 @@ import (
"strings"
"time"
"github.com/coredns/coredns/plugin/pkg/log"
"github.com/yunionio/jsonutils"
"github.com/yunionio/onecloud/pkg/cloudprovider"
"github.com/yunionio/pkg/util/secrules"
)
const (
@@ -180,3 +182,36 @@ func (self *SVpc) GetManagerId() string {
func (self *SVpc) Delete() error {
return self.region.DeleteVpc(self.VpcId)
}
func (self *SVpc) syncSecurityGroup(secgroupId string, rules []*secrules.SecurityRule) (string, error) {
secgrpId := ""
if secgroup, err := self.region.getSecurityGroupByTag(self.VpcId, secgroupId); err != nil {
if secgrpId, err = self.region.createSecurityGroup(self.VpcId, "", ""); err != nil {
return "", err
} else if err := self.region.addTagToSecurityGroup(secgrpId, "id", secgroupId, 1); err != nil {
return "", err
} else {
//addRules
log.Debugf("Add Rules: %s", secgrpId)
for _, rule := range rules {
if err := self.region.addSecurityGroupRule(secgrpId, rule); err != nil {
return "", err
}
}
}
} else {
//syncRules
log.Debugf("Sync rules %s", secgrpId)
secgrpId = secgroup.SecurityGroupId
self.region.syncSecgroupRules(secgrpId, rules)
}
return secgrpId, nil
}
func (self *SVpc) assignSecurityGroup(secgroupId string, instanceId string) error {
log.Debugf("Assign secgroup %s => %s", secgroupId, instanceId)
return self.region.assignSecurityGroup(secgroupId, instanceId)
}
func (self *SVpc) revokeSecurityGroup(secgroupId string, instanceId string) error {
return self.region.revokeSecurityGroup(secgroupId, instanceId)
}
pkg/util/k8s/k8s.go
+64
View File
@@ -0,0 +1,64 @@
package k8s
import (
"fmt"
"net/http"
"time"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
)
const (
K8sWrapTransportTimeout = 30
)
type WrapTransport func(rt http.RoundTripper) http.RoundTripper
func NewClientByFile(kubeConfigPath string, k8sWrapTransport WrapTransport) (*kubernetes.Clientset, error) {
config, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath)
if err != nil {
return nil, err
}
return kubernetes.NewForConfig(setConfigField(config, k8sWrapTransport))
}
func GetK8sClientConfig(kubeConfig []byte) (*rest.Config, error) {
var config *rest.Config
var err error
if kubeConfig != nil {
apiconfig, err := clientcmd.Load(kubeConfig)
if err != nil {
return nil, err
}
clientConfig := clientcmd.NewDefaultClientConfig(*apiconfig, &clientcmd.ConfigOverrides{})
config, err = clientConfig.ClientConfig()
if err != nil {
return nil, err
}
} else {
return nil, fmt.Errorf("kubeconfig value is nil")
}
if err != nil {
return nil, fmt.Errorf("create kubernetes config failed: %v", err)
}
return config, nil
}
func NewClientByContent(kubeConfig []byte, k8sWrapTransport WrapTransport) (*kubernetes.Clientset, error) {
config, err := GetK8sClientConfig(kubeConfig)
if err != nil {
return nil, fmt.Errorf("Create kubernetes config: %v", err)
}
return kubernetes.NewForConfig(setConfigField(config, k8sWrapTransport))
}
func setConfigField(c *rest.Config, tr WrapTransport) *rest.Config {
if tr != nil {
c.WrapTransport = tr
}
c.Timeout = time.Second * time.Duration(K8sWrapTransportTimeout)
return c
}
vendor/github.com/DataDog/dd-trace-go/LICENSE
Generated Vendored
+24
View File
@@ -0,0 +1,24 @@
Copyright (c) 2016, Datadog <info@datadoghq.com>
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
* Neither the name of Datadog nor the
names of its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
vendor/github.com/DataDog/dd-trace-go/LICENSE-3rdparty.csv
Generated Vendored
+2
View File
@@ -0,0 +1,2 @@
Component,Origin,License,Copyright
import,io.opentracing,Apache-2.0,Copyright 2016-2017 The OpenTracing Authors
1 Component Origin License Copyright
2 import io.opentracing Apache-2.0 Copyright 2016-2017 The OpenTracing Authors
vendor/github.com/DataDog/dd-trace-go/opentracing/config.go
Generated Vendored
+58
View File
@@ -0,0 +1,58 @@
package opentracing
import (
"os"
"path/filepath"
)
// Configuration struct configures the Datadog tracer. Please use the NewConfiguration
// constructor to begin.
type Configuration struct {
// Enabled, when false, returns a no-op implementation of the Tracer.
Enabled bool
// Debug, when true, writes details to logs.
Debug bool
// ServiceName specifies the name of this application.
ServiceName string
// SampleRate sets the Tracer sample rate (ext/priority.go).
SampleRate float64
// AgentHostname specifies the hostname of the agent where the traces
// are sent to.
AgentHostname string
// AgentPort specifies the port that the agent is listening on.
AgentPort string
// GlobalTags holds a set of tags that will be automatically applied to
// all spans.
GlobalTags map[string]interface{}
// TextMapPropagator is an injector used for Context propagation.
TextMapPropagator Propagator
}
// NewConfiguration creates a `Configuration` object with default values.
func NewConfiguration() *Configuration {
// default service name is the Go binary name
binaryName := filepath.Base(os.Args[0])
// Configuration struct with default values
return &Configuration{
Enabled: true,
Debug: false,
ServiceName: binaryName,
SampleRate: 1,
AgentHostname: "localhost",
AgentPort: "8126",
GlobalTags: make(map[string]interface{}),
TextMapPropagator: NewTextMapPropagator("", "", ""),
}
}
type noopCloser struct{}
func (c *noopCloser) Close() error { return nil }
vendor/github.com/DataDog/dd-trace-go/opentracing/context.go
Generated Vendored
+46
View File
@@ -0,0 +1,46 @@
package opentracing
// SpanContext represents Span state that must propagate to descendant Spans
// and across process boundaries.
type SpanContext struct {
traceID uint64
spanID uint64
parentID uint64
sampled bool
span *Span
baggage map[string]string
}
// ForeachBaggageItem grants access to all baggage items stored in the
// SpanContext
func (c SpanContext) ForeachBaggageItem(handler func(k, v string) bool) {
for k, v := range c.baggage {
if !handler(k, v) {
break
}
}
}
// WithBaggageItem returns an entirely new SpanContext with the
// given key:value baggage pair set.
func (c SpanContext) WithBaggageItem(key, val string) SpanContext {
var newBaggage map[string]string
if c.baggage == nil {
newBaggage = map[string]string{key: val}
} else {
newBaggage = make(map[string]string, len(c.baggage)+1)
for k, v := range c.baggage {
newBaggage[k] = v
}
newBaggage[key] = val
}
// Use positional parameters so the compiler will help catch new fields.
return SpanContext{
traceID: c.traceID,
spanID: c.spanID,
parentID: c.parentID,
sampled: c.sampled,
span: c.span,
baggage: newBaggage,
}
}
vendor/github.com/DataDog/dd-trace-go/opentracing/doc.go
Generated Vendored
+4
View File
@@ -0,0 +1,4 @@
// Package opentracing implements an OpenTracing (http://opentracing.io)
// compatible tracer. A Datadog tracer must be initialized through
// a Configuration object as you can see in the following examples.
package opentracing
vendor/github.com/DataDog/dd-trace-go/opentracing/propagators.go
Generated Vendored
+125
View File
@@ -0,0 +1,125 @@
package opentracing
import (
"strconv"
"strings"
ot "github.com/opentracing/opentracing-go"
)
// Propagator implementations should be able to inject and extract
// SpanContexts into an implementation specific carrier.
type Propagator interface {
// Inject takes the SpanContext and injects it into the carrier using
// an implementation specific method.
Inject(context ot.SpanContext, carrier interface{}) error
// Extract returns the SpanContext from the given carrier using an
// implementation specific method.
Extract(carrier interface{}) (ot.SpanContext, error)
}
const (
defaultBaggageHeaderPrefix = "ot-baggage-"
defaultTraceIDHeader = "x-datadog-trace-id"
defaultParentIDHeader = "x-datadog-parent-id"
)
// NewTextMapPropagator returns a new propagator which uses opentracing.TextMap
// to inject and extract values. The parameters specify the prefix that will
// be used to prefix baggage header keys along with the trace and parent header.
// Empty strings may be provided to use the defaults, which are: "ot-baggage-" as
// prefix for baggage headers, "x-datadog-trace-id" and "x-datadog-parent-id" for
// trace and parent ID headers.
func NewTextMapPropagator(baggagePrefix, traceHeader, parentHeader string) *TextMapPropagator {
if baggagePrefix == "" {
baggagePrefix = defaultBaggageHeaderPrefix
}
if traceHeader == "" {
traceHeader = defaultTraceIDHeader
}
if parentHeader == "" {
parentHeader = defaultParentIDHeader
}
return &TextMapPropagator{baggagePrefix, traceHeader, parentHeader}
}
// TextMapPropagator implements a propagator which uses opentracing.TextMap
// internally.
type TextMapPropagator struct {
baggagePrefix string
traceHeader string
parentHeader string
}
// Inject defines the TextMapPropagator to propagate SpanContext data
// out of the current process. The implementation propagates the
// TraceID and the current active SpanID, as well as the Span baggage.
func (p *TextMapPropagator) Inject(context ot.SpanContext, carrier interface{}) error {
ctx, ok := context.(SpanContext)
if !ok {
return ot.ErrInvalidSpanContext
}
writer, ok := carrier.(ot.TextMapWriter)
if !ok {
return ot.ErrInvalidCarrier
}
// propagate the TraceID and the current active SpanID
writer.Set(p.traceHeader, strconv.FormatUint(ctx.traceID, 10))
writer.Set(p.parentHeader, strconv.FormatUint(ctx.spanID, 10))
// propagate OpenTracing baggage
for k, v := range ctx.baggage {
writer.Set(p.baggagePrefix+k, v)
}
return nil
}
// Extract implements Propagator.
func (p *TextMapPropagator) Extract(carrier interface{}) (ot.SpanContext, error) {
reader, ok := carrier.(ot.TextMapReader)
if !ok {
return nil, ot.ErrInvalidCarrier
}
var err error
var traceID, parentID uint64
decodedBaggage := make(map[string]string)
// extract SpanContext fields
err = reader.ForeachKey(func(k, v string) error {
switch strings.ToLower(k) {
case p.traceHeader:
traceID, err = strconv.ParseUint(v, 10, 64)
if err != nil {
return ot.ErrSpanContextCorrupted
}
case p.parentHeader:
parentID, err = strconv.ParseUint(v, 10, 64)
if err != nil {
return ot.ErrSpanContextCorrupted
}
default:
lowercaseK := strings.ToLower(k)
if strings.HasPrefix(lowercaseK, p.baggagePrefix) {
decodedBaggage[strings.TrimPrefix(lowercaseK, p.baggagePrefix)] = v
}
}
return nil
})
if err != nil {
return nil, err
}
if traceID == 0 || parentID == 0 {
return nil, ot.ErrSpanContextNotFound
}
return SpanContext{
traceID: traceID,
spanID: parentID,
baggage: decodedBaggage,
}, nil
}
vendor/github.com/DataDog/dd-trace-go/opentracing/span.go
Generated Vendored
+151
View File
@@ -0,0 +1,151 @@
package opentracing
import (
"fmt"
"time"
ddtrace "github.com/DataDog/dd-trace-go/tracer"
ot "github.com/opentracing/opentracing-go"
"github.com/opentracing/opentracing-go/log"
)
// Span represents an active, un-finished span in the OpenTracing system.
// Spans are created by the Tracer interface.
type Span struct {
*ddtrace.Span
context SpanContext
tracer *Tracer
}
// Tracer provides access to the `Tracer`` that created this Span.
func (s *Span) Tracer() ot.Tracer {
return s.tracer
}
// Context yields the SpanContext for this Span. Note that the return
// value of Context() is still valid after a call to Span.Finish(), as is
// a call to Span.Context() after a call to Span.Finish().
func (s *Span) Context() ot.SpanContext {
return s.context
}
// SetBaggageItem sets a key:value pair on this Span and its SpanContext
// that also propagates to descendants of this Span.
func (s *Span) SetBaggageItem(key, val string) ot.Span {
s.Span.Lock()
defer s.Span.Unlock()
s.context = s.context.WithBaggageItem(key, val)
return s
}
// BaggageItem gets the value for a baggage item given its key. Returns the empty string
// if the value isn't found in this Span.
func (s *Span) BaggageItem(key string) string {
s.Span.Lock()
defer s.Span.Unlock()
return s.context.baggage[key]
}
// SetTag adds a tag to the span, overwriting pre-existing values for
// the given `key`.
func (s *Span) SetTag(key string, value interface{}) ot.Span {
switch key {
case ServiceName:
s.Span.Lock()
defer s.Span.Unlock()
s.Span.Service = fmt.Sprint(value)
case ResourceName:
s.Span.Lock()
defer s.Span.Unlock()
s.Span.Resource = fmt.Sprint(value)
case SpanType:
s.Span.Lock()
defer s.Span.Unlock()
s.Span.Type = fmt.Sprint(value)
case Error:
switch v := value.(type) {
case nil:
// no error
case error:
s.Span.SetError(v)
default:
s.Span.SetError(fmt.Errorf("%v", v))
}
default:
// NOTE: locking is not required because the `SetMeta` is
// already thread-safe
s.Span.SetMeta(key, fmt.Sprint(value))
}
return s
}
// FinishWithOptions is like Finish() but with explicit control over
// timestamps and log data.
func (s *Span) FinishWithOptions(options ot.FinishOptions) {
if options.FinishTime.IsZero() {
options.FinishTime = time.Now().UTC()
}
s.Span.FinishWithTime(options.FinishTime.UnixNano())
}
// SetOperationName sets or changes the operation name.
func (s *Span) SetOperationName(operationName string) ot.Span {
s.Span.Lock()
defer s.Span.Unlock()
s.Span.Name = operationName
return s
}
// LogFields is an efficient and type-checked way to record key:value
// logging data about a Span, though the programming interface is a little
// more verbose than LogKV().
func (s *Span) LogFields(fields ...log.Field) {
// TODO: implementation missing
}
// LogKV is a concise, readable way to record key:value logging data about
// a Span, though unfortunately this also makes it less efficient and less
// type-safe than LogFields().
func (s *Span) LogKV(keyVals ...interface{}) {
// TODO: implementation missing
}
// LogEvent is deprecated: use LogFields or LogKV
func (s *Span) LogEvent(event string) {
// TODO: implementation missing
}
// LogEventWithPayload deprecated: use LogFields or LogKV
func (s *Span) LogEventWithPayload(event string, payload interface{}) {
// TODO: implementation missing
}
// Log is deprecated: use LogFields or LogKV
func (s *Span) Log(data ot.LogData) {
// TODO: implementation missing
}
// NewSpan is the OpenTracing Span constructor
func NewSpan(operationName string) *Span {
span := &ddtrace.Span{
Name: operationName,
}
otSpan := &Span{
Span: span,
context: SpanContext{
traceID: span.TraceID,
spanID: span.SpanID,
parentID: span.ParentID,
sampled: span.Sampled,
},
}
// SpanContext is propagated and used to create children
otSpan.context.span = otSpan
return otSpan
}
vendor/github.com/DataDog/dd-trace-go/opentracing/tags.go
Generated Vendored
+12
View File
@@ -0,0 +1,12 @@
package opentracing
const (
// SpanType defines the Span type (web, db, cache)
SpanType = "span.type"
// ServiceName defines the Service name for this Span
ServiceName = "service.name"
// ResourceName defines the Resource name for the Span
ResourceName = "resource.name"
// Error defines an error.
Error = "error.error"
)
vendor/github.com/DataDog/dd-trace-go/opentracing/tracer.go
Generated Vendored
+178
View File
@@ -0,0 +1,178 @@
package opentracing
import (
"errors"
"io"
"time"
ddtrace "github.com/DataDog/dd-trace-go/tracer"
ot "github.com/opentracing/opentracing-go"
)
// Tracer is a simple, thin interface for Span creation and SpanContext
// propagation. In the current state, this Tracer is a compatibility layer
// that wraps the Datadog Tracer implementation.
type Tracer struct {
// impl is the Datadog Tracer implementation.
impl *ddtrace.Tracer
// config holds the Configuration used to create the Tracer.
config *Configuration
}
// StartSpan creates, starts, and returns a new Span with the given `operationName`
// A Span with no SpanReference options (e.g., opentracing.ChildOf() or
// opentracing.FollowsFrom()) becomes the root of its own trace.
func (t *Tracer) StartSpan(operationName string, options ...ot.StartSpanOption) ot.Span {
sso := ot.StartSpanOptions{}
for _, o := range options {
o.Apply(&sso)
}
return t.startSpanWithOptions(operationName, sso)
}
func (t *Tracer) startSpanWithOptions(operationName string, options ot.StartSpanOptions) ot.Span {
if options.StartTime.IsZero() {
options.StartTime = time.Now().UTC()
}
var context SpanContext
var hasParent bool
var parent *Span
var span *ddtrace.Span
for _, ref := range options.References {
ctx, ok := ref.ReferencedContext.(SpanContext)
if !ok {
// ignore the SpanContext since it's not valid
continue
}
// if we have parenting define it
if ref.Type == ot.ChildOfRef {
hasParent = true
context = ctx
parent = ctx.span
}
}
if parent == nil {
// create a root Span with the default service name and resource
span = t.impl.NewRootSpan(operationName, t.config.ServiceName, operationName)
if hasParent {
// the Context doesn't have a Span reference because it
// has been propagated from another process, so we set these
// values manually
span.TraceID = context.traceID
span.ParentID = context.spanID
t.impl.Sample(span)
}
} else {
// create a child Span that inherits from a parent
span = t.impl.NewChildSpan(operationName, parent.Span)
}
// create an OpenTracing compatible Span; the SpanContext has a
// back-reference that is used for parent-child hierarchy
otSpan := &Span{
Span: span,
context: SpanContext{
traceID: span.TraceID,
spanID: span.SpanID,
parentID: span.ParentID,
sampled: span.Sampled,
},
tracer: t,
}
otSpan.context.span = otSpan
// set start time
otSpan.Span.Start = options.StartTime.UnixNano()
if parent != nil {
// propagate baggage items
if l := len(parent.context.baggage); l > 0 {
otSpan.context.baggage = make(map[string]string, len(parent.context.baggage))
for k, v := range parent.context.baggage {
otSpan.context.baggage[k] = v
}
}
}
// add tags from options
for k, v := range options.Tags {
otSpan.SetTag(k, v)
}
// add global tags
for k, v := range t.config.GlobalTags {
otSpan.SetTag(k, v)
}
return otSpan
}
// Inject takes the `sm` SpanContext instance and injects it for
// propagation within `carrier`. The actual type of `carrier` depends on
// the value of `format`. Currently supported Injectors are:
// * `TextMap`
// * `HTTPHeaders`
func (t *Tracer) Inject(ctx ot.SpanContext, format interface{}, carrier interface{}) error {
switch format {
case ot.TextMap, ot.HTTPHeaders:
return t.config.TextMapPropagator.Inject(ctx, carrier)
}
return ot.ErrUnsupportedFormat
}
// Extract returns a SpanContext instance given `format` and `carrier`.
func (t *Tracer) Extract(format interface{}, carrier interface{}) (ot.SpanContext, error) {
switch format {
case ot.TextMap, ot.HTTPHeaders:
return t.config.TextMapPropagator.Extract(carrier)
}
return nil, ot.ErrUnsupportedFormat
}
// Close method implements `io.Closer` interface to graceful shutdown the Datadog
// Tracer. Note that this is a blocking operation that waits for the flushing Go
// routine.
func (t *Tracer) Close() error {
t.impl.Stop()
return nil
}
// NewTracer uses a Configuration object to initialize a Datadog Tracer.
// The initialization returns a `io.Closer` that can be used to graceful
// shutdown the tracer. If the configuration object defines a disabled
// Tracer, a no-op implementation is returned.
func NewTracer(config *Configuration) (ot.Tracer, io.Closer, error) {
if config.ServiceName == "" {
// abort initialization if a `ServiceName` is not defined
return nil, nil, errors.New("A Datadog Tracer requires a valid `ServiceName` set")
}
if config.Enabled == false {
// return a no-op implementation so Datadog provides the minimum overhead
return &ot.NoopTracer{}, &noopCloser{}, nil
}
// configure a Datadog Tracer
transport := ddtrace.NewTransport(config.AgentHostname, config.AgentPort)
tracer := &Tracer{
impl: ddtrace.NewTracerTransport(transport),
config: config,
}
tracer.impl.SetDebugLogging(config.Debug)
tracer.impl.SetSampleRate(config.SampleRate)
// set the new Datadog Tracer as a `DefaultTracer` so it can be
// used in integrations. NOTE: this is a temporary implementation
// that can be removed once all integrations have been migrated
// to the OpenTracing API.
ddtrace.DefaultTracer = tracer.impl
return tracer, tracer, nil
}
vendor/github.com/DataDog/dd-trace-go/tracer/buffer.go
Generated Vendored
+127
View File
@@ -0,0 +1,127 @@
package tracer
import (
"sync"
)
const (
// spanBufferDefaultInitSize is the initial size of our trace buffer,
// by default we allocate for a handful of spans within the trace,
// reasonable as span is actually way bigger, and avoids re-allocating
// over and over. Could be fine-tuned at runtime.
spanBufferDefaultInitSize = 10
// spanBufferDefaultMaxSize is the maximum number of spans we keep in memory.
// This is to avoid memory leaks, if above that value, spans are randomly
// dropped and ignore, resulting in corrupted tracing data, but ensuring
// original program continues to work as expected.
spanBufferDefaultMaxSize = 1e5
)
type spanBuffer struct {
channels tracerChans
spans []*Span
finishedSpans int
initSize int
maxSize int
sync.RWMutex
}
func newSpanBuffer(channels tracerChans, initSize, maxSize int) *spanBuffer {
if initSize <= 0 {
initSize = spanBufferDefaultInitSize
}
if maxSize <= 0 {
maxSize = spanBufferDefaultMaxSize
}
return &spanBuffer{
channels: channels,
initSize: initSize,
maxSize: maxSize,
}
}
func (tb *spanBuffer) Push(span *Span) {
if tb == nil {
return
}
tb.Lock()
defer tb.Unlock()
if len(tb.spans) > 0 {
// if spanBuffer is full, forget span
if len(tb.spans) >= tb.maxSize {
tb.channels.pushErr(&errorSpanBufFull{Len: len(tb.spans)})
return
}
// if there's a trace ID mismatch, ignore span
if tb.spans[0].TraceID != span.TraceID {
tb.channels.pushErr(&errorTraceIDMismatch{Expected: tb.spans[0].TraceID, Actual: span.TraceID})
return
}
}
if tb.spans == nil {
tb.spans = make([]*Span, 0, tb.initSize)
}
tb.spans = append(tb.spans, span)
}
func (tb *spanBuffer) flushable() bool {
tb.RLock()
defer tb.RUnlock()
if len(tb.spans) == 0 {
return false
}
return tb.finishedSpans == len(tb.spans)
}
func (tb *spanBuffer) ack() {
tb.Lock()
defer tb.Unlock()
tb.finishedSpans++
}
func (tb *spanBuffer) doFlush() {
if !tb.flushable() {
return
}
tb.Lock()
defer tb.Unlock()
tb.channels.pushTrace(tb.spans)
tb.spans = nil
tb.finishedSpans = 0 // important, because a buffer can be used for several flushes
}
func (tb *spanBuffer) Flush() {
if tb == nil {
return
}
tb.doFlush()
}
func (tb *spanBuffer) AckFinish() {
if tb == nil {
return
}
tb.ack()
tb.doFlush()
}
func (tb *spanBuffer) Len() int {
if tb == nil {
return 0
}
tb.RLock()
defer tb.RUnlock()
return len(tb.spans)
}
vendor/github.com/DataDog/dd-trace-go/tracer/channels.go
Generated Vendored
+88
View File
@@ -0,0 +1,88 @@
package tracer
const (
// traceChanLen is the capacity of the trace channel. This channels is emptied
// on a regular basis (worker thread) or when it reaches 50% of its capacity.
// If it's full, then data is simply dropped and ignored, with a log message.
// This only happens under heavy load,
traceChanLen = 1000
// serviceChanLen is the length of the service channel. As for the trace channel,
// it's emptied by worker thread or when it reaches 50%. Note that there should
// be much less data here, as service data does not be to be updated that often.
serviceChanLen = 50
// errChanLen is the number of errors we keep in the error channel. When this
// one is full, errors are just ignored, dropped, nothing left. At some point,
// there's already a whole lot of errors in the backlog, there's no real point
// in keeping millions of errors, a representative sample is enough. And we
// don't want to block user code and/or bloat memory or log files with redundant data.
errChanLen = 200
)
// traceChans holds most tracer channels together, it's mostly used to
// pass them together to the span buffer/context. It's obviously safe
// to access it concurrently as it contains channels only. And it's convenient
// to have it isolated from tracer, for the sake of unit testing.
type tracerChans struct {
trace chan []*Span
service chan Service
err chan error
traceFlush chan struct{}
serviceFlush chan struct{}
errFlush chan struct{}
}
func newTracerChans() tracerChans {
return tracerChans{
trace: make(chan []*Span, traceChanLen),
service: make(chan Service, serviceChanLen),
err: make(chan error, errChanLen),
traceFlush: make(chan struct{}, 1),
serviceFlush: make(chan struct{}, 1),
errFlush: make(chan struct{}, 1),
}
}
func (tc *tracerChans) pushTrace(trace []*Span) {
if len(tc.trace) >= cap(tc.trace)/2 { // starts being full, anticipate, try and flush soon
select {
case tc.traceFlush <- struct{}{}:
default: // a flush was already requested, skip
}
}
select {
case tc.trace <- trace:
default: // never block user code
tc.pushErr(&errorTraceChanFull{Len: len(tc.trace)})
}
}
func (tc *tracerChans) pushService(service Service) {
if len(tc.service) >= cap(tc.service)/2 { // starts being full, anticipate, try and flush soon
select {
case tc.serviceFlush <- struct{}{}:
default: // a flush was already requested, skip
}
}
select {
case tc.service <- service:
default: // never block user code
tc.pushErr(&errorServiceChanFull{Len: len(tc.service)})
}
}
func (tc *tracerChans) pushErr(err error) {
if len(tc.err) >= cap(tc.err)/2 { // starts being full, anticipate, try and flush soon
select {
case tc.errFlush <- struct{}{}:
default: // a flush was already requested, skip
}
}
select {
case tc.err <- err:
default:
// OK, if we get this, our error error buffer is full,
// we can assume it is filled with meaningful messages which
// are going to be logged and hopefully read, nothing better
// we can do, blocking would make things worse.
}
}
vendor/github.com/DataDog/dd-trace-go/tracer/context.go
Generated Vendored
+42
View File
@@ -0,0 +1,42 @@
package tracer
import (
"context"
)
var spanKey = "datadog_trace_span"
// ContextWithSpan will return a new context that includes the given span.
// DEPRECATED: use span.Context(ctx) instead.
func ContextWithSpan(ctx context.Context, span *Span) context.Context {
if span == nil {
return ctx
}
return span.Context(ctx)
}
// SpanFromContext returns the stored *Span from the Context if it's available.
// This helper returns also the ok value that is true if the span is present.
func SpanFromContext(ctx context.Context) (*Span, bool) {
if ctx == nil {
return nil, false
}
span, ok := ctx.Value(spanKey).(*Span)
return span, ok
}
// SpanFromContextDefault returns the stored *Span from the Context. If not, it
// will return an empty span that will do nothing.
func SpanFromContextDefault(ctx context.Context) *Span {
// FIXME[matt] is it better to return a singleton empty span?
if ctx == nil {
return &Span{}
}
span, ok := SpanFromContext(ctx)
if !ok {
return &Span{}
}
return span
}
vendor/github.com/DataDog/dd-trace-go/tracer/doc.go
Generated Vendored
+19
View File
@@ -0,0 +1,19 @@
// Package tracer contains Datadog's tracing client. It is used to trace
// requests as they flow across web servers, databases and microservices so
// that developers have visibility into bottlenecks and troublesome
// requests.
//
// Package tracer has two core objects: Tracers and Spans. Spans represent
// a chunk of computation time. They have names, durations, timestamps and
// other metadata. Tracers are used to create hierarchies of spans in a
// request, buffer and submit them to the server.
//
// The tracing client can perform trace sampling. While the trace agent
// already samples traces to reduce bandwidth usage, client sampling reduces
// performance overhead.
//
// To enable APM and/or tracing of supported integrations, follow the instructions for
// the appropriate package: https://godoc.org/github.com/DataDog/dd-trace-go/tracer#pkg-subdirectories
//
// Sample code is available in the two examples below:
package tracer
vendor/github.com/DataDog/dd-trace-go/tracer/encoder.go
Generated Vendored
+114
View File
@@ -0,0 +1,114 @@
package tracer
import (
"bytes"
"encoding/json"
"github.com/ugorji/go/codec"
)
const (
jsonContentType = "application/json"
msgpackContentType = "application/msgpack"
)
// Encoder is a generic interface that expects encoding methods for traces and
// services, and a Read() method that will be used by the http handler
type Encoder interface {
EncodeTraces(traces [][]*Span) error
EncodeServices(services map[string]Service) error
Read(p []byte) (int, error)
ContentType() string
}
var mh codec.MsgpackHandle
// msgpackEncoder encodes a list of traces in Msgpack format
type msgpackEncoder struct {
buffer *bytes.Buffer
encoder *codec.Encoder
contentType string
}
func newMsgpackEncoder() *msgpackEncoder {
buffer := &bytes.Buffer{}
encoder := codec.NewEncoder(buffer, &mh)
return &msgpackEncoder{
buffer: buffer,
encoder: encoder,
contentType: msgpackContentType,
}
}
// EncodeTraces serializes the given trace list into the internal buffer,
// returning the error if any.
func (e *msgpackEncoder) EncodeTraces(traces [][]*Span) error {
return e.encoder.Encode(traces)
}
// EncodeServices serializes a service map into the internal buffer.
func (e *msgpackEncoder) EncodeServices(services map[string]Service) error {
return e.encoder.Encode(services)
}
// Read values from the internal buffer
func (e *msgpackEncoder) Read(p []byte) (int, error) {
return e.buffer.Read(p)
}
// ContentType return the msgpackEncoder content-type
func (e *msgpackEncoder) ContentType() string {
return e.contentType
}
// jsonEncoder encodes a list of traces in JSON format
type jsonEncoder struct {
buffer *bytes.Buffer
encoder *json.Encoder
contentType string
}
// newJSONEncoder returns a new encoder for the JSON format.
func newJSONEncoder() *jsonEncoder {
buffer := &bytes.Buffer{}
encoder := json.NewEncoder(buffer)
return &jsonEncoder{
buffer: buffer,
encoder: encoder,
contentType: jsonContentType,
}
}
// EncodeTraces serializes the given trace list into the internal buffer,
// returning the error if any.
func (e *jsonEncoder) EncodeTraces(traces [][]*Span) error {
return e.encoder.Encode(traces)
}
// EncodeServices serializes a service map into the internal buffer.
func (e *jsonEncoder) EncodeServices(services map[string]Service) error {
return e.encoder.Encode(services)
}
// Read values from the internal buffer
func (e *jsonEncoder) Read(p []byte) (int, error) {
return e.buffer.Read(p)
}
// ContentType return the jsonEncoder content-type
func (e *jsonEncoder) ContentType() string {
return e.contentType
}
// encoderFactory will provide a new encoder each time we want to flush traces or services.
type encoderFactory func() Encoder
func jsonEncoderFactory() Encoder {
return newJSONEncoder()
}
func msgpackEncoderFactory() Encoder {
return newMsgpackEncoder()
}
vendor/github.com/DataDog/dd-trace-go/tracer/errors.go
Generated Vendored
+157
View File
@@ -0,0 +1,157 @@
package tracer
import (
"log"
"strconv"
)
const (
errorPrefix = "Datadog Tracer Error: "
)
// errorSpanBufFull is raised when there's no more room in the buffer
type errorSpanBufFull struct {
// Len is the length of the buffer (which is full)
Len int
}
// Error provides a readable error message.
func (e *errorSpanBufFull) Error() string {
return "span buffer is full (length: " + strconv.Itoa(e.Len) + ")"
}
// errorTraceChanFull is raised when there's no more room in the channel
type errorTraceChanFull struct {
// Len is the length of the channel (which is full)
Len int
}
// Error provides a readable error message.
func (e *errorTraceChanFull) Error() string {
return "trace channel is full (length: " + strconv.Itoa(e.Len) + ")"
}
// errorServiceChanFull is raised when there's no more room in the channel
type errorServiceChanFull struct {
// Len is the length of the channel (which is full)
Len int
}
// Error provides a readable error message.
func (e *errorServiceChanFull) Error() string {
return "service channel is full (length: " + strconv.Itoa(e.Len) + ")"
}
// errorTraceIDMismatch is raised when a trying to put a span in the wrong place.
type errorTraceIDMismatch struct {
// Expected is the trace ID we should have.
Expected uint64
// Actual is the trace ID we have and is wrong.
Actual uint64
}
// Error provides a readable error message.
func (e *errorTraceIDMismatch) Error() string {
return "trace ID mismatch (expected: " +
strconv.FormatUint(e.Expected, 16) +
" actual: " +
strconv.FormatUint(e.Actual, 16) +
")"
}
// errorNoSpanBuf is raised when trying to finish/push a span that has no buffer associated to it.
type errorNoSpanBuf struct {
// SpanName is the name of the span which could not be pushed (hint for the log reader).
SpanName string
}
// Error provides a readable error message.
func (e *errorNoSpanBuf) Error() string {
return "no span buffer (span name: '" + e.SpanName + "')"
}
// errorFlushLostTraces is raised when trying to finish/push a span that has no buffer associated to it.
type errorFlushLostTraces struct {
// Nb is the number of traces lost in that flush
Nb int
}
// Error provides a readable error message.
func (e *errorFlushLostTraces) Error() string {
return "unable to flush traces, lost " + strconv.Itoa(e.Nb) + " traces"
}
// errorFlushLostServices is raised when trying to finish/push a span that has no buffer associated to it.
type errorFlushLostServices struct {
// Nb is the number of services lost in that flush
Nb int
}
// Error provides a readable error message.
func (e *errorFlushLostServices) Error() string {
return "unable to flush services, lost " + strconv.Itoa(e.Nb) + " services"
}
type errorSummary struct {
Count int
Example string
}
// errorKey returns a unique key for each error type
func errorKey(err error) string {
if err == nil {
return ""
}
switch err.(type) {
case *errorSpanBufFull:
return "ErrorSpanBufFull"
case *errorTraceChanFull:
return "ErrorTraceChanFull"
case *errorServiceChanFull:
return "ErrorServiceChanFull"
case *errorTraceIDMismatch:
return "ErrorTraceIDMismatch"
case *errorNoSpanBuf:
return "ErrorNoSpanBuf"
case *errorFlushLostTraces:
return "ErrorFlushLostTraces"
case *errorFlushLostServices:
return "ErrorFlushLostServices"
}
return err.Error() // possibly high cardinality, but this is unexpected
}
func aggregateErrors(errChan <-chan error) map[string]errorSummary {
errs := make(map[string]errorSummary, len(errChan))
for {
select {
case err := <-errChan:
if err != nil { // double-checking, we don't want to panic here...
key := errorKey(err)
summary := errs[key]
summary.Count++
summary.Example = err.Error()
errs[key] = summary
}
default: // stop when there's no more data
return errs
}
}
}
// logErrors logs the errors, preventing log file flooding, when there
// are many messages, it caps them and shows a quick summary.
// As of today it only logs using standard golang log package, but
// later we could send those stats to agent [TODO:christian].
func logErrors(errChan <-chan error) {
errs := aggregateErrors(errChan)
for _, v := range errs {
var repeat string
if v.Count > 1 {
repeat = " (repeated " + strconv.Itoa(v.Count) + " times)"
}
log.Println(errorPrefix + v.Example + repeat)
}
}
vendor/github.com/DataDog/dd-trace-go/tracer/ext/app_types.go
Generated Vendored
+9
View File
@@ -0,0 +1,9 @@
package ext
// Application types for services.
const (
AppTypeWeb = "web"
AppTypeDB = "db"
AppTypeCache = "cache"
AppTypeRPC = "rpc"
)
vendor/github.com/DataDog/dd-trace-go/tracer/ext/cassandra.go
Generated Vendored
+11
View File
@@ -0,0 +1,11 @@
package ext
const (
CassandraType = "cassandra"
CassandraQuery = "cassandra.query"
CassandraConsistencyLevel = "cassandra.consistency_level"
CassandraCluster = "cassandra.cluster"
CassandraRowCount = "cassandra.row_count"
CassandraKeyspace = "cassandra.keyspace"
CassandraPaginated = "cassandra.paginated"
)
vendor/github.com/DataDog/dd-trace-go/tracer/ext/errors.go
Generated Vendored
+8
View File
@@ -0,0 +1,8 @@
package ext
// Standard error message metadata fields.
const (
ErrorMsg = "error.msg"
ErrorType = "error.type"
ErrorStack = "error.stack"
)
vendor/github.com/DataDog/dd-trace-go/tracer/ext/http.go
Generated Vendored
+9
View File
@@ -0,0 +1,9 @@
package ext
// HTTP meta constants.
const (
HTTPType = "http"
HTTPMethod = "http.method"
HTTPCode = "http.status_code"
HTTPURL = "http.url"
)
vendor/github.com/DataDog/dd-trace-go/tracer/ext/net.go
Generated Vendored
+6
View File
@@ -0,0 +1,6 @@
package ext
const (
TargetHost = "out.host"
TargetPort = "out.port"
)
vendor/github.com/DataDog/dd-trace-go/tracer/ext/priority.go
Generated Vendored
+19
View File
@@ -0,0 +1,19 @@
package ext
// Priority is a hint given to the backend so that it knows which traces to reject or kept.
// In a distributed context, it should be set before any context propagation (fork, RPC calls) to be effective.
const (
// PriorityUserReject informs the backend that a trace should be rejected and not stored.
// This should be used by user code overriding default priority.
PriorityUserReject = -1
// PriorityAutoReject informs the backend that a trace should be rejected and not stored.
// This is used by the builtin sampler.
PriorityAutoReject = 0
// PriorityAutoReject informs the backend that a trace should be kept and not stored.
// This is used by the builtin sampler.
PriorityAutoKeep = 1
// PriorityUserReject informs the backend that a trace should be kept and not stored.
// This should be used by user code overriding default priority.
PriorityUserKeep = 2
)
vendor/github.com/DataDog/dd-trace-go/tracer/ext/sql.go
Generated Vendored
+6
View File
@@ -0,0 +1,6 @@
package ext
const (
SQLType = "sql"
SQLQuery = "sql.query"
)
vendor/github.com/DataDog/dd-trace-go/tracer/ext/system.go
Generated Vendored
+7
View File
@@ -0,0 +1,7 @@
package ext
// Standard system metadata names
const (
// The pid of the traced process
Pid = "system.pid"
)
vendor/github.com/DataDog/dd-trace-go/tracer/ext/tracer.go
Generated Vendored
+14
View File
@@ -0,0 +1,14 @@
package ext
import (
"runtime"
"strings"
)
const (
Lang = "go"
Interpreter = runtime.Compiler + "-" + runtime.GOARCH + "-" + runtime.GOOS
TracerVersion = "v0.5.0"
)
var LangVersion = strings.TrimPrefix(runtime.Version(), Lang)
vendor/github.com/DataDog/dd-trace-go/tracer/rand.go
Generated Vendored
+50
View File
@@ -0,0 +1,50 @@
package tracer
import (
cryptorand "crypto/rand"
"log"
"math"
"math/big"
"math/rand"
"sync"
"time"
)
// randGen is the global thread safe random number generator
var randGen *rand.Rand
type randSource struct {
source rand.Source
sync.Mutex
}
func newRandSource() *randSource {
var seed int64
max := big.NewInt(math.MaxInt64)
n, err := cryptorand.Int(cryptorand.Reader, max)
if err == nil {
seed = n.Int64()
} else {
log.Printf("%scannot generate random seed: %v; using current time\n", errorPrefix, err)
seed = time.Now().UnixNano()
}
source := rand.NewSource(seed)
return &randSource{source: source}
}
func (rs *randSource) Int63() int64 {
rs.Lock()
n := rs.source.Int63()
rs.Unlock()
return n
}
func (rs *randSource) Seed(seed int64) {
rs.Lock()
rs.Seed(seed)
rs.Unlock()
}
vendor/github.com/DataDog/dd-trace-go/tracer/sampler.go
Generated Vendored
+57
View File
@@ -0,0 +1,57 @@
package tracer
const (
// sampleRateMetricKey is the metric key holding the applied sample rate. Has to be the same as the Agent.
sampleRateMetricKey = "_sample_rate"
// constants used for the Knuth hashing, same constants as the Agent.
maxTraceID = ^uint64(0)
maxTraceIDFloat = float64(maxTraceID)
samplerHasher = uint64(1111111111111111111)
)
// sampler is the generic interface of any sampler
type sampler interface {
Sample(span *Span) // Tells if a trace is sampled and sets `span.Sampled`
}
// allSampler samples all the traces
type allSampler struct{}
func newAllSampler() *allSampler {
return &allSampler{}
}
// Sample samples a span
func (s *allSampler) Sample(span *Span) {
// Nothing to do here, since by default a trace is sampled
}
// rateSampler samples from a sample rate
type rateSampler struct {
SampleRate float64
}
// newRateSampler returns an initialized rateSampler with its sample rate
func newRateSampler(sampleRate float64) *rateSampler {
return &rateSampler{
SampleRate: sampleRate,
}
}
// Sample samples a span
func (s *rateSampler) Sample(span *Span) {
if s.SampleRate < 1 {
span.Sampled = sampleByRate(span.TraceID, s.SampleRate)
span.SetMetric(sampleRateMetricKey, s.SampleRate)
}
}
// sampleByRate tells if a trace (from its ID) with a given rate should be sampled.
// Its implementation has to be the same as the Trace Agent.
func sampleByRate(traceID uint64, sampleRate float64) bool {
if sampleRate < 1 {
return traceID*samplerHasher < uint64(sampleRate*maxTraceIDFloat)
}
return true
}
vendor/github.com/DataDog/dd-trace-go/tracer/span.go
Generated Vendored
+338
View File
@@ -0,0 +1,338 @@
package tracer
import (
"context"
"fmt"
"reflect"
"runtime/debug"
"strings"
"sync"
"time"
)
const (
errorMsgKey = "error.msg"
errorTypeKey = "error.type"
errorStackKey = "error.stack"
samplingPriorityKey = "_sampling_priority_v1"
)
// Span represents a computation. Callers must call Finish when a span is
// complete to ensure it's submitted.
//
// span := tracer.NewRootSpan("web.request", "datadog.com", "/user/{id}")
// defer span.Finish() // or FinishWithErr(err)
//
// In general, spans should be created with the tracer.NewSpan* functions,
// so they will be submitted on completion.
type Span struct {
// Name is the name of the operation being measured. Some examples
// might be "http.handler", "fileserver.upload" or "video.decompress".
// Name should be set on every span.
Name string `json:"name"`
// Service is the name of the process doing a particular job. Some
// examples might be "user-database" or "datadog-web-app". Services
// will be inherited from parents, so only set this in your app's
// top level span.
Service string `json:"service"`
// Resource is a query to a service. A web application might use
// resources like "/user/{user_id}". A sql database might use resources
// like "select * from user where id = ?".
//
// You can track thousands of resources (not millions or billions) so
// prefer normalized resources like "/user/{id}" to "/user/123".
//
// Resources should only be set on an app's top level spans.
Resource string `json:"resource"`
Type string `json:"type"` // protocol associated with the span
Start int64 `json:"start"` // span start time expressed in nanoseconds since epoch
Duration int64 `json:"duration"` // duration of the span expressed in nanoseconds
Meta map[string]string `json:"meta,omitempty"` // arbitrary map of metadata
Metrics map[string]float64 `json:"metrics,omitempty"` // arbitrary map of numeric metrics
SpanID uint64 `json:"span_id"` // identifier of this span
TraceID uint64 `json:"trace_id"` // identifier of the root span
ParentID uint64 `json:"parent_id"` // identifier of the span's direct parent
Error int32 `json:"error"` // error status of the span; 0 means no errors
Sampled bool `json:"-"` // if this span is sampled (and should be kept/recorded) or not
sync.RWMutex
tracer *Tracer // the tracer that generated this span
finished bool // true if the span has been submitted to a tracer.
// parent contains a link to the parent. In most cases, ParentID can be inferred from this.
// However, ParentID can technically be overridden (typical usage: distributed tracing)
// and also, parent == nil is used to identify root and top-level ("local root") spans.
parent *Span
buffer *spanBuffer
}
// NewSpan creates a new span. This is a low-level function, required for testing and advanced usage.
// Most of the time one should prefer the Tracer NewRootSpan or NewChildSpan methods.
func NewSpan(name, service, resource string, spanID, traceID, parentID uint64, tracer *Tracer) *Span {
return &Span{
Name: name,
Service: service,
Resource: resource,
Meta: tracer.getAllMeta(),
SpanID: spanID,
TraceID: traceID,
ParentID: parentID,
Start: now(),
Sampled: true,
tracer: tracer,
}
}
// setMeta adds an arbitrary meta field to the current Span. The span
// must be locked outside of this function
func (s *Span) setMeta(key, value string) {
if s == nil {
return
}
// We don't lock spans when flushing, so we could have a data race when
// modifying a span as it's being flushed. This protects us against that
// race, since spans are marked `finished` before we flush them.
if s.finished {
return
}
if s.Meta == nil {
s.Meta = make(map[string]string)
}
s.Meta[key] = value
}
// SetMeta adds an arbitrary meta field to the current Span.
// If the Span has been finished, it will not be modified by the method.
func (s *Span) SetMeta(key, value string) {
if s == nil {
return
}
s.Lock()
defer s.Unlock()
s.setMeta(key, value)
}
// SetMetas adds arbitrary meta fields from a given map to the current Span.
// If the Span has been finished, it will not be modified by the method.
func (s *Span) SetMetas(metas map[string]string) {
for k, v := range metas {
s.SetMeta(k, v)
}
}
// GetMeta will return the value for the given tag or the empty string if it
// doesn't exist.
func (s *Span) GetMeta(key string) string {
if s == nil {
return ""
}
s.RLock()
defer s.RUnlock()
if s.Meta == nil {
return ""
}
return s.Meta[key]
}
// SetMetrics adds a metric field to the current Span.
// DEPRECATED: Use SetMetric
func (s *Span) SetMetrics(key string, value float64) {
if s == nil {
return
}
s.SetMetric(key, value)
}
// SetMetric sets a float64 value for the given key. It acts
// like `set_meta()` and it simply add a tag without further processing.
// This method doesn't create a Datadog metric.
func (s *Span) SetMetric(key string, val float64) {
if s == nil {
return
}
s.Lock()
defer s.Unlock()
// We don't lock spans when flushing, so we could have a data race when
// modifying a span as it's being flushed. This protects us against that
// race, since spans are marked `finished` before we flush them.
if s.finished {
return
}
if s.Metrics == nil {
s.Metrics = make(map[string]float64)
}
s.Metrics[key] = val
}
// SetError stores an error object within the span meta. The Error status is
// updated and the error.Error() string is included with a default meta key.
// If the Span has been finished, it will not be modified by this method.
func (s *Span) SetError(err error) {
if err == nil || s == nil {
return
}
s.Lock()
defer s.Unlock()
// We don't lock spans when flushing, so we could have a data race when
// modifying a span as it's being flushed. This protects us against that
// race, since spans are marked `finished` before we flush them.
if s.finished {
return
}
s.Error = 1
s.setMeta(errorMsgKey, err.Error())
s.setMeta(errorTypeKey, reflect.TypeOf(err).String())
stack := debug.Stack()
s.setMeta(errorStackKey, string(stack))
}
// Finish closes this Span (but not its children) providing the duration
// of this part of the tracing session. This method is idempotent so
// calling this method multiple times is safe and doesn't update the
// current Span. Once a Span has been finished, methods that modify the Span
// will become no-ops.
func (s *Span) Finish() {
s.finish(now())
}
// FinishWithTime closes this Span at the given `finishTime`. The
// behavior is the same as `Finish()`.
func (s *Span) FinishWithTime(finishTime int64) {
s.finish(finishTime)
}
func (s *Span) finish(finishTime int64) {
if s == nil {
return
}
s.Lock()
finished := s.finished
if !finished {
if s.Duration == 0 {
s.Duration = finishTime - s.Start
}
s.finished = true
}
s.Unlock()
if finished {
// no-op, called twice, no state change...
return
}
if s.buffer == nil {
if s.tracer != nil {
s.tracer.channels.pushErr(&errorNoSpanBuf{SpanName: s.Name})
}
return
}
// If tracer is explicitely disabled, stop now
if s.tracer != nil && !s.tracer.Enabled() {
return
}
// If not sampled, drop it
if !s.Sampled {
return
}
s.buffer.AckFinish() // put data in channel only if trace is completely finished
// It's important that when Finish() exits, the data is put in
// the channel for real, when the trace is finished.
// Otherwise, tests could become flaky (because you never know in what state
// the channel is).
}
// FinishWithErr marks a span finished and sets the given error if it's
// non-nil.
func (s *Span) FinishWithErr(err error) {
if s == nil {
return
}
s.SetError(err)
s.Finish()
}
// String returns a human readable representation of the span. Not for
// production, just debugging.
func (s *Span) String() string {
lines := []string{
fmt.Sprintf("Name: %s", s.Name),
fmt.Sprintf("Service: %s", s.Service),
fmt.Sprintf("Resource: %s", s.Resource),
fmt.Sprintf("TraceID: %d", s.TraceID),
fmt.Sprintf("SpanID: %d", s.SpanID),
fmt.Sprintf("ParentID: %d", s.ParentID),
fmt.Sprintf("Start: %s", time.Unix(0, s.Start)),
fmt.Sprintf("Duration: %s", time.Duration(s.Duration)),
fmt.Sprintf("Error: %d", s.Error),
fmt.Sprintf("Type: %s", s.Type),
"Tags:",
}
s.RLock()
for key, val := range s.Meta {
lines = append(lines, fmt.Sprintf("\t%s:%s", key, val))
}
s.RUnlock()
return strings.Join(lines, "\n")
}
// Context returns a copy of the given context that includes this span.
// This span can be accessed downstream with SpanFromContext and friends.
func (s *Span) Context(ctx context.Context) context.Context {
if s == nil {
return ctx
}
return context.WithValue(ctx, spanKey, s)
}
// Tracer returns the tracer that created this span.
func (s *Span) Tracer() *Tracer {
if s == nil {
return nil
}
return s.tracer
}
// SetSamplingPriority sets the sampling priority.
func (s *Span) SetSamplingPriority(priority int) {
s.SetMetric(samplingPriorityKey, float64(priority))
}
// HasSamplingPriority returns true if sampling priority is set.
// It can be defined to either zero or non-zero.
func (s *Span) HasSamplingPriority() bool {
_, hasSamplingPriority := s.Metrics[samplingPriorityKey]
return hasSamplingPriority
}
// GetSamplingPriority gets the sampling priority.
func (s *Span) GetSamplingPriority() int {
return int(s.Metrics[samplingPriorityKey])
}
// NextSpanID returns a new random span id.
func NextSpanID() uint64 {
return uint64(randGen.Int63())
}
vendor/github.com/DataDog/dd-trace-go/tracer/time.go
Generated Vendored
+10
View File
@@ -0,0 +1,10 @@
// +build !windows
package tracer
import "time"
// now returns current UTC time in nanos.
func now() int64 {
return time.Now().UTC().UnixNano()
}
vendor/github.com/DataDog/dd-trace-go/tracer/time_windows.go
Generated Vendored
+34
View File
@@ -0,0 +1,34 @@
package tracer
import (
"golang.org/x/sys/windows"
"log"
"time"
)
// This method is more precise than the go1.8 time.Now on Windows
// See https://msdn.microsoft.com/en-us/library/windows/desktop/hh706895(v=vs.85).aspx
// It is however ~10x slower and requires Windows 8+.
func highPrecisionNow() int64 {
var ft windows.Filetime
windows.GetSystemTimePreciseAsFileTime(&ft)
return ft.Nanoseconds()
}
func lowPrecisionNow() int64 {
return time.Now().UTC().UnixNano()
}
var now func() int64
// If GetSystemTimePreciseAsFileTime is not available we default to the less
// precise implementation based on time.Now()
func init() {
if err := windows.LoadGetSystemTimePreciseAsFileTime(); err != nil {
log.Printf("Unable to load high precison timer, defaulting to time.Now()")
now = lowPrecisionNow
} else {
log.Printf("Using high precision timer")
now = highPrecisionNow
}
}
vendor/github.com/DataDog/dd-trace-go/tracer/tracer.go
Generated Vendored
+429
View File
@@ -0,0 +1,429 @@
package tracer
import (
"context"
"log"
"math/rand"
"os"
"strconv"
"sync"
"time"
"github.com/DataDog/dd-trace-go/tracer/ext"
)
const (
flushInterval = 2 * time.Second
)
func init() {
randGen = rand.New(newRandSource())
}
type Service struct {
Name string `json:"-"` // the internal of the service (e.g. acme_search, datadog_web)
App string `json:"app"` // the name of the application (e.g. rails, postgres, custom-app)
AppType string `json:"app_type"` // the type of the application (e.g. db, web)
}
func (s Service) Equal(s2 Service) bool {
return s.Name == s2.Name && s.App == s2.App && s.AppType == s2.AppType
}
// Tracer creates, buffers and submits Spans which are used to time blocks of
// compuration.
//
// When a tracer is disabled, it will not submit spans for processing.
type Tracer struct {
transport Transport // is the transport mechanism used to delivery spans to the agent
sampler sampler // is the trace sampler to only keep some samples
debugMu sync.RWMutex // protects the debugLoggingEnabled attribute while allowing concurrent reads
debugLoggingEnabled bool
enableMu sync.RWMutex
enabled bool // defines if the Tracer is enabled or not
meta map[string]string
metaMu sync.RWMutex
channels tracerChans
services map[string]Service // name -> service
exit chan struct{}
exitWG *sync.WaitGroup
forceFlushIn chan struct{}
forceFlushOut chan struct{}
}
// NewTracer creates a new Tracer. Most users should use the package's
// DefaultTracer instance.
func NewTracer() *Tracer {
return NewTracerTransport(newDefaultTransport())
}
// NewTracerTransport create a new Tracer with the given transport.
func NewTracerTransport(transport Transport) *Tracer {
t := &Tracer{
enabled: true,
transport: transport,
sampler: newAllSampler(),
debugLoggingEnabled: false,
channels: newTracerChans(),
services: make(map[string]Service),
exit: make(chan struct{}),
exitWG: &sync.WaitGroup{},
forceFlushIn: make(chan struct{}, 0), // must be size 0 (blocking)
forceFlushOut: make(chan struct{}, 0), // must be size 0 (blocking)
}
// start a background worker
t.exitWG.Add(1)
go t.worker()
return t
}
// Stop stops the tracer.
func (t *Tracer) Stop() {
close(t.exit)
t.exitWG.Wait()
}
// SetEnabled will enable or disable the tracer.
func (t *Tracer) SetEnabled(enabled bool) {
t.enableMu.Lock()
defer t.enableMu.Unlock()
t.enabled = enabled
}
// Enabled returns whether or not a tracer is enabled.
func (t *Tracer) Enabled() bool {
t.enableMu.RLock()
defer t.enableMu.RUnlock()
return t.enabled
}
// SetSampleRate sets a sample rate for all the future traces.
// sampleRate has to be between 0.0 and 1.0 and represents the ratio of traces
// that will be sampled. 0.0 means that the tracer won't send any trace. 1.0
// means that the tracer will send all traces.
func (t *Tracer) SetSampleRate(sampleRate float64) {
if sampleRate == 1 {
t.sampler = newAllSampler()
} else if sampleRate >= 0 && sampleRate < 1 {
t.sampler = newRateSampler(sampleRate)
} else {
log.Printf("tracer.SetSampleRate rate must be between 0 and 1, now: %f", sampleRate)
}
}
// SetServiceInfo update the application and application type for the given
// service.
func (t *Tracer) SetServiceInfo(name, app, appType string) {
t.channels.pushService(Service{
Name: name,
App: app,
AppType: appType,
})
}
// SetMeta adds an arbitrary meta field at the tracer level.
// This will append those tags to each span created by the tracer.
func (t *Tracer) SetMeta(key, value string) {
if t == nil { // Defensive, span could be initialized with nil tracer
return
}
t.metaMu.Lock()
if t.meta == nil {
t.meta = make(map[string]string)
}
t.meta[key] = value
t.metaMu.Unlock()
}
// getAllMeta returns all the meta set by this tracer.
// In most cases, it is nil.
func (t *Tracer) getAllMeta() map[string]string {
if t == nil { // Defensive, span could be initialized with nil tracer
return nil
}
var meta map[string]string
t.metaMu.RLock()
if t.meta != nil {
meta = make(map[string]string, len(t.meta))
for key, value := range t.meta {
meta[key] = value
}
}
t.metaMu.RUnlock()
return meta
}
// NewRootSpan creates a span with no parent. Its ids will be randomly
// assigned.
func (t *Tracer) NewRootSpan(name, service, resource string) *Span {
spanID := NextSpanID()
span := NewSpan(name, service, resource, spanID, spanID, 0, t)
span.buffer = newSpanBuffer(t.channels, 0, 0)
t.Sample(span)
// [TODO:christian] introduce distributed sampling here
span.buffer.Push(span)
// Add the process id to all root spans
span.SetMeta(ext.Pid, strconv.Itoa(os.Getpid()))
return span
}
// NewChildSpan returns a new span that is child of the Span passed as
// argument.
func (t *Tracer) NewChildSpan(name string, parent *Span) *Span {
spanID := NextSpanID()
// when we're using parenting in inner functions, it's possible that
// a nil pointer is sent to this function as argument. To prevent a crash,
// it's better to be defensive and to produce a wrongly configured span
// that is not sent to the trace agent.
if parent == nil {
span := NewSpan(name, "", name, spanID, spanID, spanID, t)
span.buffer = newSpanBuffer(t.channels, 0, 0)
t.Sample(span)
// [TODO:christian] introduce distributed sampling here
span.buffer.Push(span)
return span
}
parent.RLock()
// child that is correctly configured
span := NewSpan(name, parent.Service, name, spanID, parent.TraceID, parent.SpanID, parent.tracer)
// child sampling same as the parent
span.Sampled = parent.Sampled
if parent.HasSamplingPriority() {
span.SetSamplingPriority(parent.GetSamplingPriority())
}
span.parent = parent
span.buffer = parent.buffer
parent.RUnlock()
span.buffer.Push(span)
return span
}
// NewChildSpanFromContext will create a child span of the span contained in
// the given context. If the context contains no span, an empty span will be
// returned.
func (t *Tracer) NewChildSpanFromContext(name string, ctx context.Context) *Span {
span, _ := SpanFromContext(ctx) // tolerate nil spans
return t.NewChildSpan(name, span)
}
// NewChildSpanWithContext will create and return a child span of the span contained in the given
// context, as well as a copy of the parent context containing the created
// child span. If the context contains no span, an empty root span will be returned.
// If nil is passed in for the context, a context will be created.
func (t *Tracer) NewChildSpanWithContext(name string, ctx context.Context) (*Span, context.Context) {
span := t.NewChildSpanFromContext(name, ctx)
return span, span.Context(ctx)
}
// SetDebugLogging will set the debug level
func (t *Tracer) SetDebugLogging(debug bool) {
t.debugMu.Lock()
defer t.debugMu.Unlock()
t.debugLoggingEnabled = debug
}
// DebugLoggingEnabled returns true if the debug level is enabled and false otherwise.
func (t *Tracer) DebugLoggingEnabled() bool {
t.debugMu.RLock()
defer t.debugMu.RUnlock()
return t.debugLoggingEnabled
}
func (t *Tracer) getTraces() [][]*Span {
traces := make([][]*Span, 0, len(t.channels.trace))
for {
select {
case trace := <-t.channels.trace:
traces = append(traces, trace)
default: // return when there's no more data
return traces
}
}
}
// flushTraces will push any currently buffered traces to the server.
func (t *Tracer) flushTraces() {
traces := t.getTraces()
if t.DebugLoggingEnabled() {
log.Printf("Sending %d traces", len(traces))
for _, trace := range traces {
if len(trace) > 0 {
log.Printf("TRACE: %d\n", trace[0].TraceID)
for _, span := range trace {
log.Printf("SPAN:\n%s", span.String())
}
}
}
}
// bal if there's nothing to do
if !t.Enabled() || t.transport == nil || len(traces) == 0 {
return
}
_, err := t.transport.SendTraces(traces)
if err != nil {
t.channels.pushErr(err)
t.channels.pushErr(&errorFlushLostTraces{Nb: len(traces)}) // explicit log messages with nb of lost traces
}
}
func (t *Tracer) updateServices() bool {
servicesModified := false
for {
select {
case service := <-t.channels.service:
if s, found := t.services[service.Name]; !found || !s.Equal(service) {
t.services[service.Name] = service
servicesModified = true
}
default: // return when there's no more data
return servicesModified
}
}
}
// flushTraces will push any currently buffered services to the server.
func (t *Tracer) flushServices() {
servicesModified := t.updateServices()
if !t.Enabled() || !servicesModified {
return
}
_, err := t.transport.SendServices(t.services)
if err != nil {
t.channels.pushErr(err)
t.channels.pushErr(&errorFlushLostServices{Nb: len(t.services)}) // explicit log messages with nb of lost services
}
}
// flushErrs will process log messages that were queued
func (t *Tracer) flushErrs() {
logErrors(t.channels.err)
}
func (t *Tracer) flush() {
t.flushTraces()
t.flushServices()
t.flushErrs()
}
// ForceFlush forces a flush of data (traces and services) to the agent.
// Flushes are done by a background task on a regular basis, so you never
// need to call this manually, mostly useful for testing and debugging.
func (t *Tracer) ForceFlush() {
t.forceFlushIn <- struct{}{}
<-t.forceFlushOut
}
// Sample samples a span with the internal sampler.
func (t *Tracer) Sample(span *Span) {
t.sampler.Sample(span)
}
// worker periodically flushes traces and services to the transport.
func (t *Tracer) worker() {
defer t.exitWG.Done()
flushTicker := time.NewTicker(flushInterval)
defer flushTicker.Stop()
for {
select {
case <-flushTicker.C:
t.flush()
case <-t.forceFlushIn:
t.flush()
t.forceFlushOut <- struct{}{} // caller blocked until this is done
case <-t.channels.traceFlush:
t.flushTraces()
case <-t.channels.serviceFlush:
t.flushServices()
case <-t.channels.errFlush:
t.flushErrs()
case <-t.exit:
t.flush()
return
}
}
}
// DefaultTracer is a global tracer that is enabled by default. All of the
// packages top level NewSpan functions use the default tracer.
//
// span := tracer.NewRootSpan("sql.query", "user-db", "select * from foo where id = ?")
// defer span.Finish()
//
var DefaultTracer = NewTracer()
// NewRootSpan creates a span with no parent. Its ids will be randomly
// assigned.
func NewRootSpan(name, service, resource string) *Span {
return DefaultTracer.NewRootSpan(name, service, resource)
}
// NewChildSpan creates a span that is a child of parent. It will inherit the
// parent's service and resource.
func NewChildSpan(name string, parent *Span) *Span {
return DefaultTracer.NewChildSpan(name, parent)
}
// NewChildSpanFromContext will create a child span of the span contained in
// the given context. If the context contains no span, a span with
// no service or resource will be returned.
func NewChildSpanFromContext(name string, ctx context.Context) *Span {
return DefaultTracer.NewChildSpanFromContext(name, ctx)
}
// NewChildSpanWithContext will create and return a child span of the span contained in the given
// context, as well as a copy of the parent context containing the created
// child span. If the context contains no span, an empty root span will be returned.
// If nil is passed in for the context, a context will be created.
func NewChildSpanWithContext(name string, ctx context.Context) (*Span, context.Context) {
return DefaultTracer.NewChildSpanWithContext(name, ctx)
}
// Enable will enable the default tracer.
func Enable() {
DefaultTracer.SetEnabled(true)
}
// Disable will disable the default tracer.
func Disable() {
DefaultTracer.SetEnabled(false)
}
vendor/github.com/DataDog/dd-trace-go/tracer/transport.go
Generated Vendored
+195
View File
@@ -0,0 +1,195 @@
package tracer
import (
"errors"
"fmt"
"log"
"net/http"
"strconv"
"time"
"github.com/DataDog/dd-trace-go/tracer/ext"
)
const (
defaultHostname = "localhost"
defaultPort = "8126"
defaultHTTPTimeout = time.Second // defines the current timeout before giving up with the send process
traceCountHeader = "X-Datadog-Trace-Count" // header containing the number of traces in the payload
)
// Transport is an interface for span submission to the agent.
type Transport interface {
SendTraces(spans [][]*Span) (*http.Response, error)
SendServices(services map[string]Service) (*http.Response, error)
SetHeader(key, value string)
}
// NewTransport returns a new Transport implementation that sends traces to a
// trace agent running on the given hostname and port. If the zero values for
// hostname and port are provided, the default values will be used ("localhost"
// for hostname, and "8126" for port).
//
// In general, using this method is only necessary if you have a trace agent
// running on a non-default port or if it's located on another machine.
func NewTransport(hostname, port string) Transport {
if hostname == "" {
hostname = defaultHostname
}
if port == "" {
port = defaultPort
}
return newHTTPTransport(hostname, port)
}
// newDefaultTransport return a default transport for this tracing client
func newDefaultTransport() Transport {
return newHTTPTransport(defaultHostname, defaultPort)
}
type httpTransport struct {
traceURL string // the delivery URL for traces
legacyTraceURL string // the legacy delivery URL for traces
serviceURL string // the delivery URL for services
legacyServiceURL string // the legacy delivery URL for services
client *http.Client // the HTTP client used in the POST
headers map[string]string // the Transport headers
compatibilityMode bool // the Agent targets a legacy API for compatibility reasons
// [WARNING] We tried to reuse encoders thanks to a pool, but that led us to having race conditions.
// Indeed, when we send the encoder as the request body, the persistConn.writeLoop() goroutine
// can theoretically read the underlying buffer whereas the encoder has been returned to the pool.
// Since the underlying bytes.Buffer is not thread safe, this can make the app panicking.
// since this method will later on spawn a goroutine referencing this buffer.
// That's why we prefer the less performant yet SAFE implementation of allocating a new encoder every time we flush.
getEncoder encoderFactory
}
// newHTTPTransport returns an httpTransport for the given endpoint
func newHTTPTransport(hostname, port string) *httpTransport {
// initialize the default EncoderPool with Encoder headers
defaultHeaders := map[string]string{
"Datadog-Meta-Lang": ext.Lang,
"Datadog-Meta-Lang-Version": ext.LangVersion,
"Datadog-Meta-Lang-Interpreter": ext.Interpreter,
"Datadog-Meta-Tracer-Version": ext.TracerVersion,
}
return &httpTransport{
traceURL: fmt.Sprintf("http://%s:%s/v0.3/traces", hostname, port),
legacyTraceURL: fmt.Sprintf("http://%s:%s/v0.2/traces", hostname, port),
serviceURL: fmt.Sprintf("http://%s:%s/v0.3/services", hostname, port),
legacyServiceURL: fmt.Sprintf("http://%s:%s/v0.2/services", hostname, port),
getEncoder: msgpackEncoderFactory,
client: &http.Client{
Timeout: defaultHTTPTimeout,
},
headers: defaultHeaders,
compatibilityMode: false,
}
}
func (t *httpTransport) SendTraces(traces [][]*Span) (*http.Response, error) {
if t.traceURL == "" {
return nil, errors.New("provided an empty URL, giving up")
}
encoder := t.getEncoder()
// encode the spans and return the error if any
err := encoder.EncodeTraces(traces)
if err != nil {
return nil, err
}
// prepare the client and send the payload
req, _ := http.NewRequest("POST", t.traceURL, encoder)
for header, value := range t.headers {
req.Header.Set(header, value)
}
req.Header.Set(traceCountHeader, strconv.Itoa(len(traces)))
req.Header.Set("Content-Type", encoder.ContentType())
response, err := t.client.Do(req)
// if we have an error, return an empty Response to protect against nil pointer dereference
if err != nil {
return &http.Response{StatusCode: 0}, err
}
defer response.Body.Close()
// if we got a 404 we should downgrade the API to a stable version (at most once)
if (response.StatusCode == 404 || response.StatusCode == 415) && !t.compatibilityMode {
log.Printf("calling the endpoint '%s' but received %d; downgrading the API\n", t.traceURL, response.StatusCode)
t.apiDowngrade()
return t.SendTraces(traces)
}
if sc := response.StatusCode; sc != 200 {
return response, fmt.Errorf("SendTraces expected response code 200, received %v", sc)
}
return response, err
}
func (t *httpTransport) SendServices(services map[string]Service) (*http.Response, error) {
if t.serviceURL == "" {
return nil, errors.New("provided an empty URL, giving up")
}
encoder := t.getEncoder()
if err := encoder.EncodeServices(services); err != nil {
return nil, err
}
// Send it
req, err := http.NewRequest("POST", t.serviceURL, encoder)
if err != nil {
return nil, fmt.Errorf("cannot create http request: %v", err)
}
for header, value := range t.headers {
req.Header.Set(header, value)
}
req.Header.Set("Content-Type", encoder.ContentType())
response, err := t.client.Do(req)
if err != nil {
return &http.Response{StatusCode: 0}, err
}
defer response.Body.Close()
// Downgrade if necessary
if (response.StatusCode == 404 || response.StatusCode == 415) && !t.compatibilityMode {
log.Printf("calling the endpoint '%s' but received %d; downgrading the API\n", t.traceURL, response.StatusCode)
t.apiDowngrade()
return t.SendServices(services)
}
if sc := response.StatusCode; sc != 200 {
return response, fmt.Errorf("SendServices expected response code 200, received %v", sc)
}
return response, err
}
// SetHeader sets the internal header for the httpTransport
func (t *httpTransport) SetHeader(key, value string) {
t.headers[key] = value
}
// changeEncoder switches the encoder so that a different API with different
// format can be targeted, preventing failures because of outdated agents
func (t *httpTransport) changeEncoder(encoderFactory encoderFactory) {
t.getEncoder = encoderFactory
}
// apiDowngrade downgrades the used encoder and API level. This method must fallback to a safe
// encoder and API, so that it will success despite users' configurations. This action
// ensures that the compatibility mode is activated so that the downgrade will be
// executed only once.
func (t *httpTransport) apiDowngrade() {
t.compatibilityMode = true
t.traceURL = t.legacyTraceURL
t.serviceURL = t.legacyServiceURL
t.changeEncoder(jsonEncoderFactory)
}
vendor/github.com/Shopify/sarama/.gitignore
Generated
+26
View File
@@ -0,0 +1,26 @@
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
*.test
# Folders
_obj
_test
.vagrant
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
coverage.txt
vendor/github.com/Shopify/sarama/.travis.yml
Generated Vendored
+36
View File
@@ -0,0 +1,36 @@
language: go
go:
- 1.8.x
- 1.9.x
- 1.10.x
env:
global:
- KAFKA_PEERS=localhost:9091,localhost:9092,localhost:9093,localhost:9094,localhost:9095
- TOXIPROXY_ADDR=http://localhost:8474
- KAFKA_INSTALL_ROOT=/home/travis/kafka
- KAFKA_HOSTNAME=localhost
- DEBUG=true
matrix:
- KAFKA_VERSION=0.11.0.2
- KAFKA_VERSION=1.0.0
- KAFKA_VERSION=1.1.0
before_install:
- export REPOSITORY_ROOT=${TRAVIS_BUILD_DIR}
- vagrant/install_cluster.sh
- vagrant/boot_cluster.sh
- vagrant/create_topics.sh
install: make install_dependencies
script:
- make test
- make vet
- make errcheck
- make fmt
after_success:
- bash <(curl -s https://codecov.io/bash)
after_script: vagrant/halt_cluster.sh
vendor/github.com/Shopify/sarama/CHANGELOG.md
Generated Vendored
+541
View File
@@ -0,0 +1,541 @@
# Changelog
#### Version 1.17.0 (2018-05-30)
New Features:
- Add support for gzip compression levels
([#1044](https://github.com/Shopify/sarama/pull/1044)).
- Add support for Metadata request/response pairs versions v1 to v5
([#1047](https://github.com/Shopify/sarama/pull/1047),
[#1069](https://github.com/Shopify/sarama/pull/1069)).
- Add versioning to JoinGroup request/response pairs
([#1098](https://github.com/Shopify/sarama/pull/1098))
- Add support for CreatePartitions, DeleteGroups, DeleteRecords request/response pairs
([#1065](https://github.com/Shopify/sarama/pull/1065),
[#1096](https://github.com/Shopify/sarama/pull/1096),
[#1027](https://github.com/Shopify/sarama/pull/1027)).
- Add `Controller()` method to Client interface
([#1063](https://github.com/Shopify/sarama/pull/1063)).
Improvements:
- ConsumerMetadataReq/Resp has been migrated to FindCoordinatorReq/Resp
([#1010](https://github.com/Shopify/sarama/pull/1010)).
- Expose missing protocol parts: `msgSet` and `recordBatch`
([#1049](https://github.com/Shopify/sarama/pull/1049)).
- Add support for v1 DeleteTopics Request
([#1052](https://github.com/Shopify/sarama/pull/1052)).
- Add support for Go 1.10
([#1064](https://github.com/Shopify/sarama/pull/1064)).
- Claim support for Kafka 1.1.0
([#1073](https://github.com/Shopify/sarama/pull/1073)).
Bug Fixes:
- Fix FindCoordinatorResponse.encode to allow nil Coordinator
([#1050](https://github.com/Shopify/sarama/pull/1050),
[#1051](https://github.com/Shopify/sarama/pull/1051)).
- Clear all metadata when we have the latest topic info
([#1033](https://github.com/Shopify/sarama/pull/1033)).
- Make `PartitionConsumer.Close` idempotent
([#1092](https://github.com/Shopify/sarama/pull/1092)).
#### Version 1.16.0 (2018-02-12)
New Features:
- Add support for the Create/Delete Topics request/response pairs
([#1007](https://github.com/Shopify/sarama/pull/1007),
[#1008](https://github.com/Shopify/sarama/pull/1008)).
- Add support for the Describe/Create/Delete ACL request/response pairs
([#1009](https://github.com/Shopify/sarama/pull/1009)).
- Add support for the five transaction-related request/response pairs
([#1016](https://github.com/Shopify/sarama/pull/1016)).
Improvements:
- Permit setting version on mock producer responses
([#999](https://github.com/Shopify/sarama/pull/999)).
- Add `NewMockBrokerListener` helper for testing TLS connections
([#1019](https://github.com/Shopify/sarama/pull/1019)).
- Changed the default value for `Consumer.Fetch.Default` from 32KiB to 1MiB
which results in much higher throughput in most cases
([#1024](https://github.com/Shopify/sarama/pull/1024)).
- Reuse the `time.Ticker` across fetch requests in the PartitionConsumer to
reduce CPU and memory usage when processing many partitions
([#1028](https://github.com/Shopify/sarama/pull/1028)).
- Assign relative offsets to messages in the producer to save the brokers a
recompression pass
([#1002](https://github.com/Shopify/sarama/pull/1002),
[#1015](https://github.com/Shopify/sarama/pull/1015)).
Bug Fixes:
- Fix producing uncompressed batches with the new protocol format
([#1032](https://github.com/Shopify/sarama/issues/1032)).
- Fix consuming compacted topics with the new protocol format
([#1005](https://github.com/Shopify/sarama/issues/1005)).
- Fix consuming topics with a mix of protocol formats
([#1021](https://github.com/Shopify/sarama/issues/1021)).
- Fix consuming when the broker includes multiple batches in a single response
([#1022](https://github.com/Shopify/sarama/issues/1022)).
- Fix detection of `PartialTrailingMessage` when the partial message was
truncated before the magic value indicating its version
([#1030](https://github.com/Shopify/sarama/pull/1030)).
- Fix expectation-checking in the mock of `SyncProducer.SendMessages`
([#1035](https://github.com/Shopify/sarama/pull/1035)).
#### Version 1.15.0 (2017-12-08)
New Features:
- Claim official support for Kafka 1.0, though it did already work
([#984](https://github.com/Shopify/sarama/pull/984)).
- Helper methods for Kafka version numbers to/from strings
([#989](https://github.com/Shopify/sarama/pull/989)).
- Implement CreatePartitions request/response
([#985](https://github.com/Shopify/sarama/pull/985)).
Improvements:
- Add error codes 45-60
([#986](https://github.com/Shopify/sarama/issues/986)).
Bug Fixes:
- Fix slow consuming for certain Kafka 0.11/1.0 configurations
([#982](https://github.com/Shopify/sarama/pull/982)).
- Correctly determine when a FetchResponse contains the new message format
([#990](https://github.com/Shopify/sarama/pull/990)).
- Fix producing with multiple headers
([#996](https://github.com/Shopify/sarama/pull/996)).
- Fix handling of truncated record batches
([#998](https://github.com/Shopify/sarama/pull/998)).
- Fix leaking metrics when closing brokers
([#991](https://github.com/Shopify/sarama/pull/991)).
#### Version 1.14.0 (2017-11-13)
New Features:
- Add support for the new Kafka 0.11 record-batch format, including the wire
protocol and the necessary behavioural changes in the producer and consumer.
Transactions and idempotency are not yet supported, but producing and
consuming should work with all the existing bells and whistles (batching,
compression, etc) as well as the new custom headers. Thanks to Vlad Hanciuta
of Arista Networks for this work. Part of
([#901](https://github.com/Shopify/sarama/issues/901)).
Bug Fixes:
- Fix encoding of ProduceResponse versions in test
([#970](https://github.com/Shopify/sarama/pull/970)).
- Return partial replicas list when we have it
([#975](https://github.com/Shopify/sarama/pull/975)).
#### Version 1.13.0 (2017-10-04)
New Features:
- Support for FetchRequest version 3
([#905](https://github.com/Shopify/sarama/pull/905)).
- Permit setting version on mock FetchResponses
([#939](https://github.com/Shopify/sarama/pull/939)).
- Add a configuration option to support storing only minimal metadata for
extremely large clusters
([#937](https://github.com/Shopify/sarama/pull/937)).
- Add `PartitionOffsetManager.ResetOffset` for backtracking tracked offsets
([#932](https://github.com/Shopify/sarama/pull/932)).
Improvements:
- Provide the block-level timestamp when consuming compressed messages
([#885](https://github.com/Shopify/sarama/issues/885)).
- `Client.Replicas` and `Client.InSyncReplicas` now respect the order returned
by the broker, which can be meaningful
([#930](https://github.com/Shopify/sarama/pull/930)).
- Use a `Ticker` to reduce consumer timer overhead at the cost of higher
variance in the actual timeout
([#933](https://github.com/Shopify/sarama/pull/933)).
Bug Fixes:
- Gracefully handle messages with negative timestamps
([#907](https://github.com/Shopify/sarama/pull/907)).
- Raise a proper error when encountering an unknown message version
([#940](https://github.com/Shopify/sarama/pull/940)).
#### Version 1.12.0 (2017-05-08)
New Features:
- Added support for the `ApiVersions` request and response pair, and Kafka
version 0.10.2 ([#867](https://github.com/Shopify/sarama/pull/867)). Note
that you still need to specify the Kafka version in the Sarama configuration
for the time being.
- Added a `Brokers` method to the Client which returns the complete set of
active brokers ([#813](https://github.com/Shopify/sarama/pull/813)).
- Added an `InSyncReplicas` method to the Client which returns the set of all
in-sync broker IDs for the given partition, now that the Kafka versions for
which this was misleading are no longer in our supported set
([#872](https://github.com/Shopify/sarama/pull/872)).
- Added a `NewCustomHashPartitioner` method which allows constructing a hash
partitioner with a custom hash method in case the default (FNV-1a) is not
suitable
([#837](https://github.com/Shopify/sarama/pull/837),
[#841](https://github.com/Shopify/sarama/pull/841)).
Improvements:
- Recognize more Kafka error codes
([#859](https://github.com/Shopify/sarama/pull/859)).
Bug Fixes:
- Fix an issue where decoding a malformed FetchRequest would not return the
correct error ([#818](https://github.com/Shopify/sarama/pull/818)).
- Respect ordering of group protocols in JoinGroupRequests. This fix is
transparent if you're using the `AddGroupProtocol` or
`AddGroupProtocolMetadata` helpers; otherwise you will need to switch from
the `GroupProtocols` field (now deprecated) to use `OrderedGroupProtocols`
([#812](https://github.com/Shopify/sarama/issues/812)).
- Fix an alignment-related issue with atomics on 32-bit architectures
([#859](https://github.com/Shopify/sarama/pull/859)).
#### Version 1.11.0 (2016-12-20)
_Important:_ As of Sarama 1.11 it is necessary to set the config value of
`Producer.Return.Successes` to true in order to use the SyncProducer. Previous
versions would silently override this value when instantiating a SyncProducer
which led to unexpected values and data races.
New Features:
- Metrics! Thanks to Sébastien Launay for all his work on this feature
([#701](https://github.com/Shopify/sarama/pull/701),
[#746](https://github.com/Shopify/sarama/pull/746),
[#766](https://github.com/Shopify/sarama/pull/766)).
- Add support for LZ4 compression
([#786](https://github.com/Shopify/sarama/pull/786)).
- Add support for ListOffsetRequest v1 and Kafka 0.10.1
([#775](https://github.com/Shopify/sarama/pull/775)).
- Added a `HighWaterMarks` method to the Consumer which aggregates the
`HighWaterMarkOffset` values of its child topic/partitions
([#769](https://github.com/Shopify/sarama/pull/769)).
Bug Fixes:
- Fixed producing when using timestamps, compression and Kafka 0.10
([#759](https://github.com/Shopify/sarama/pull/759)).
- Added missing decoder methods to DescribeGroups response
([#756](https://github.com/Shopify/sarama/pull/756)).
- Fix producer shutdown when `Return.Errors` is disabled
([#787](https://github.com/Shopify/sarama/pull/787)).
- Don't mutate configuration in SyncProducer
([#790](https://github.com/Shopify/sarama/pull/790)).
- Fix crash on SASL initialization failure
([#795](https://github.com/Shopify/sarama/pull/795)).
#### Version 1.10.1 (2016-08-30)
Bug Fixes:
- Fix the documentation for `HashPartitioner` which was incorrect
([#717](https://github.com/Shopify/sarama/pull/717)).
- Permit client creation even when it is limited by ACLs
([#722](https://github.com/Shopify/sarama/pull/722)).
- Several fixes to the consumer timer optimization code, regressions introduced
in v1.10.0. Go's timers are finicky
([#730](https://github.com/Shopify/sarama/pull/730),
[#733](https://github.com/Shopify/sarama/pull/733),
[#734](https://github.com/Shopify/sarama/pull/734)).
- Handle consuming compressed relative offsets with Kafka 0.10
([#735](https://github.com/Shopify/sarama/pull/735)).
#### Version 1.10.0 (2016-08-02)
_Important:_ As of Sarama 1.10 it is necessary to tell Sarama the version of
Kafka you are running against (via the `config.Version` value) in order to use
features that may not be compatible with old Kafka versions. If you don't
specify this value it will default to 0.8.2 (the minimum supported), and trying
to use more recent features (like the offset manager) will fail with an error.
_Also:_ The offset-manager's behaviour has been changed to match the upstream
java consumer (see [#705](https://github.com/Shopify/sarama/pull/705) and
[#713](https://github.com/Shopify/sarama/pull/713)). If you use the
offset-manager, please ensure that you are committing one *greater* than the
last consumed message offset or else you may end up consuming duplicate
messages.
New Features:
- Support for Kafka 0.10
([#672](https://github.com/Shopify/sarama/pull/672),
[#678](https://github.com/Shopify/sarama/pull/678),
[#681](https://github.com/Shopify/sarama/pull/681), and others).
- Support for configuring the target Kafka version
([#676](https://github.com/Shopify/sarama/pull/676)).
- Batch producing support in the SyncProducer
([#677](https://github.com/Shopify/sarama/pull/677)).
- Extend producer mock to allow setting expectations on message contents
([#667](https://github.com/Shopify/sarama/pull/667)).
Improvements:
- Support `nil` compressed messages for deleting in compacted topics
([#634](https://github.com/Shopify/sarama/pull/634)).
- Pre-allocate decoding errors, greatly reducing heap usage and GC time against
misbehaving brokers ([#690](https://github.com/Shopify/sarama/pull/690)).
- Re-use consumer expiry timers, removing one allocation per consumed message
([#707](https://github.com/Shopify/sarama/pull/707)).
Bug Fixes:
- Actually default the client ID to "sarama" like we say we do
([#664](https://github.com/Shopify/sarama/pull/664)).
- Fix a rare issue where `Client.Leader` could return the wrong error
([#685](https://github.com/Shopify/sarama/pull/685)).
- Fix a possible tight loop in the consumer
([#693](https://github.com/Shopify/sarama/pull/693)).
- Match upstream's offset-tracking behaviour
([#705](https://github.com/Shopify/sarama/pull/705)).
- Report UnknownTopicOrPartition errors from the offset manager
([#706](https://github.com/Shopify/sarama/pull/706)).
- Fix possible negative partition value from the HashPartitioner
([#709](https://github.com/Shopify/sarama/pull/709)).
#### Version 1.9.0 (2016-05-16)
New Features:
- Add support for custom offset manager retention durations
([#602](https://github.com/Shopify/sarama/pull/602)).
- Publish low-level mocks to enable testing of third-party producer/consumer
implementations ([#570](https://github.com/Shopify/sarama/pull/570)).
- Declare support for Golang 1.6
([#611](https://github.com/Shopify/sarama/pull/611)).
- Support for SASL plain-text auth
([#648](https://github.com/Shopify/sarama/pull/648)).
Improvements:
- Simplified broker locking scheme slightly
([#604](https://github.com/Shopify/sarama/pull/604)).
- Documentation cleanup
([#605](https://github.com/Shopify/sarama/pull/605),
[#621](https://github.com/Shopify/sarama/pull/621),
[#654](https://github.com/Shopify/sarama/pull/654)).
Bug Fixes:
- Fix race condition shutting down the OffsetManager
([#658](https://github.com/Shopify/sarama/pull/658)).
#### Version 1.8.0 (2016-02-01)
New Features:
- Full support for Kafka 0.9:
- All protocol messages and fields
([#586](https://github.com/Shopify/sarama/pull/586),
[#588](https://github.com/Shopify/sarama/pull/588),
[#590](https://github.com/Shopify/sarama/pull/590)).
- Verified that TLS support works
([#581](https://github.com/Shopify/sarama/pull/581)).
- Fixed the OffsetManager compatibility
([#585](https://github.com/Shopify/sarama/pull/585)).
Improvements:
- Optimize for fewer system calls when reading from the network
([#584](https://github.com/Shopify/sarama/pull/584)).
- Automatically retry `InvalidMessage` errors to match upstream behaviour
([#589](https://github.com/Shopify/sarama/pull/589)).
#### Version 1.7.0 (2015-12-11)
New Features:
- Preliminary support for Kafka 0.9
([#572](https://github.com/Shopify/sarama/pull/572)). This comes with several
caveats:
- Protocol-layer support is mostly in place
([#577](https://github.com/Shopify/sarama/pull/577)), however Kafka 0.9
renamed some messages and fields, which we did not in order to preserve API
compatibility.
- The producer and consumer work against 0.9, but the offset manager does
not ([#573](https://github.com/Shopify/sarama/pull/573)).
- TLS support may or may not work
([#581](https://github.com/Shopify/sarama/pull/581)).
Improvements:
- Don't wait for request timeouts on dead brokers, greatly speeding recovery
when the TCP connection is left hanging
([#548](https://github.com/Shopify/sarama/pull/548)).
- Refactored part of the producer. The new version provides a much more elegant
solution to [#449](https://github.com/Shopify/sarama/pull/449). It is also
slightly more efficient, and much more precise in calculating batch sizes
when compression is used
([#549](https://github.com/Shopify/sarama/pull/549),
[#550](https://github.com/Shopify/sarama/pull/550),
[#551](https://github.com/Shopify/sarama/pull/551)).
Bug Fixes:
- Fix race condition in consumer test mock
([#553](https://github.com/Shopify/sarama/pull/553)).
#### Version 1.6.1 (2015-09-25)
Bug Fixes:
- Fix panic that could occur if a user-supplied message value failed to encode
([#449](https://github.com/Shopify/sarama/pull/449)).
#### Version 1.6.0 (2015-09-04)
New Features:
- Implementation of a consumer offset manager using the APIs introduced in
Kafka 0.8.2. The API is designed mainly for integration into a future
high-level consumer, not for direct use, although it is *possible* to use it
directly.
([#461](https://github.com/Shopify/sarama/pull/461)).
Improvements:
- CRC32 calculation is much faster on machines with SSE4.2 instructions,
removing a major hotspot from most profiles
([#255](https://github.com/Shopify/sarama/pull/255)).
Bug Fixes:
- Make protocol decoding more robust against some malformed packets generated
by go-fuzz ([#523](https://github.com/Shopify/sarama/pull/523),
[#525](https://github.com/Shopify/sarama/pull/525)) or found in other ways
([#528](https://github.com/Shopify/sarama/pull/528)).
- Fix a potential race condition panic in the consumer on shutdown
([#529](https://github.com/Shopify/sarama/pull/529)).
#### Version 1.5.0 (2015-08-17)
New Features:
- TLS-encrypted network connections are now supported. This feature is subject
to change when Kafka releases built-in TLS support, but for now this is
enough to work with TLS-terminating proxies
([#154](https://github.com/Shopify/sarama/pull/154)).
Improvements:
- The consumer will not block if a single partition is not drained by the user;
all other partitions will continue to consume normally
([#485](https://github.com/Shopify/sarama/pull/485)).
- Formatting of error strings has been much improved
([#495](https://github.com/Shopify/sarama/pull/495)).
- Internal refactoring of the producer for code cleanliness and to enable
future work ([#300](https://github.com/Shopify/sarama/pull/300)).
Bug Fixes:
- Fix a potential deadlock in the consumer on shutdown
([#475](https://github.com/Shopify/sarama/pull/475)).
#### Version 1.4.3 (2015-07-21)
Bug Fixes:
- Don't include the partitioner in the producer's "fetch partitions"
circuit-breaker ([#466](https://github.com/Shopify/sarama/pull/466)).
- Don't retry messages until the broker is closed when abandoning a broker in
the producer ([#468](https://github.com/Shopify/sarama/pull/468)).
- Update the import path for snappy-go, it has moved again and the API has
changed slightly ([#486](https://github.com/Shopify/sarama/pull/486)).
#### Version 1.4.2 (2015-05-27)
Bug Fixes:
- Update the import path for snappy-go, it has moved from google code to github
([#456](https://github.com/Shopify/sarama/pull/456)).
#### Version 1.4.1 (2015-05-25)
Improvements:
- Optimizations when decoding snappy messages, thanks to John Potocny
([#446](https://github.com/Shopify/sarama/pull/446)).
Bug Fixes:
- Fix hypothetical race conditions on producer shutdown
([#450](https://github.com/Shopify/sarama/pull/450),
[#451](https://github.com/Shopify/sarama/pull/451)).
#### Version 1.4.0 (2015-05-01)
New Features:
- The consumer now implements `Topics()` and `Partitions()` methods to enable
users to dynamically choose what topics/partitions to consume without
instantiating a full client
([#431](https://github.com/Shopify/sarama/pull/431)).
- The partition-consumer now exposes the high water mark offset value returned
by the broker via the `HighWaterMarkOffset()` method ([#339](https://github.com/Shopify/sarama/pull/339)).
- Added a `kafka-console-consumer` tool capable of handling multiple
partitions, and deprecated the now-obsolete `kafka-console-partitionConsumer`
([#439](https://github.com/Shopify/sarama/pull/439),
[#442](https://github.com/Shopify/sarama/pull/442)).
Improvements:
- The producer's logging during retry scenarios is more consistent, more
useful, and slightly less verbose
([#429](https://github.com/Shopify/sarama/pull/429)).
- The client now shuffles its initial list of seed brokers in order to prevent
thundering herd on the first broker in the list
([#441](https://github.com/Shopify/sarama/pull/441)).
Bug Fixes:
- The producer now correctly manages its state if retries occur when it is
shutting down, fixing several instances of confusing behaviour and at least
one potential deadlock ([#419](https://github.com/Shopify/sarama/pull/419)).
- The consumer now handles messages for different partitions asynchronously,
making it much more resilient to specific user code ordering
([#325](https://github.com/Shopify/sarama/pull/325)).
#### Version 1.3.0 (2015-04-16)
New Features:
- The client now tracks consumer group coordinators using
ConsumerMetadataRequests similar to how it tracks partition leadership using
regular MetadataRequests ([#411](https://github.com/Shopify/sarama/pull/411)).
This adds two methods to the client API:
- `Coordinator(consumerGroup string) (*Broker, error)`
- `RefreshCoordinator(consumerGroup string) error`
Improvements:
- ConsumerMetadataResponses now automatically create a Broker object out of the
ID/address/port combination for the Coordinator; accessing the fields
individually has been deprecated
([#413](https://github.com/Shopify/sarama/pull/413)).
- Much improved handling of `OffsetOutOfRange` errors in the consumer.
Consumers will fail to start if the provided offset is out of range
([#418](https://github.com/Shopify/sarama/pull/418))
and they will automatically shut down if the offset falls out of range
([#424](https://github.com/Shopify/sarama/pull/424)).
- Small performance improvement in encoding and decoding protocol messages
([#427](https://github.com/Shopify/sarama/pull/427)).
Bug Fixes:
- Fix a rare race condition in the client's background metadata refresher if
it happens to be activated while the client is being closed
([#422](https://github.com/Shopify/sarama/pull/422)).
#### Version 1.2.0 (2015-04-07)
Improvements:
- The producer's behaviour when `Flush.Frequency` is set is now more intuitive
([#389](https://github.com/Shopify/sarama/pull/389)).
- The producer is now somewhat more memory-efficient during and after retrying
messages due to an improved queue implementation
([#396](https://github.com/Shopify/sarama/pull/396)).
- The consumer produces much more useful logging output when leadership
changes ([#385](https://github.com/Shopify/sarama/pull/385)).
- The client's `GetOffset` method will now automatically refresh metadata and
retry once in the event of stale information or similar
([#394](https://github.com/Shopify/sarama/pull/394)).
- Broker connections now have support for using TCP keepalives
([#407](https://github.com/Shopify/sarama/issues/407)).
Bug Fixes:
- The OffsetCommitRequest message now correctly implements all three possible
API versions ([#390](https://github.com/Shopify/sarama/pull/390),
[#400](https://github.com/Shopify/sarama/pull/400)).
#### Version 1.1.0 (2015-03-20)
Improvements:
- Wrap the producer's partitioner call in a circuit-breaker so that repeatedly
broken topics don't choke throughput
([#373](https://github.com/Shopify/sarama/pull/373)).
Bug Fixes:
- Fix the producer's internal reference counting in certain unusual scenarios
([#367](https://github.com/Shopify/sarama/pull/367)).
- Fix the consumer's internal reference counting in certain unusual scenarios
([#369](https://github.com/Shopify/sarama/pull/369)).
- Fix a condition where the producer's internal control messages could have
gotten stuck ([#368](https://github.com/Shopify/sarama/pull/368)).
- Fix an issue where invalid partition lists would be cached when asking for
metadata for a non-existant topic ([#372](https://github.com/Shopify/sarama/pull/372)).
#### Version 1.0.0 (2015-03-17)
Version 1.0.0 is the first tagged version, and is almost a complete rewrite. The primary differences with previous untagged versions are:
- The producer has been rewritten; there is now a `SyncProducer` with a blocking API, and an `AsyncProducer` that is non-blocking.
- The consumer has been rewritten to only open one connection per broker instead of one connection per partition.
- The main types of Sarama are now interfaces to make depedency injection easy; mock implementations for `Consumer`, `SyncProducer` and `AsyncProducer` are provided in the `github.com/Shopify/sarama/mocks` package.
- For most uses cases, it is no longer necessary to open a `Client`; this will be done for you.
- All the configuration values have been unified in the `Config` struct.
- Much improved test suite.
vendor/github.com/Shopify/sarama/LICENSE
Generated Vendored
+20
View File
@@ -0,0 +1,20 @@
Copyright (c) 2013 Shopify
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
vendor/github.com/Shopify/sarama/Makefile
Generated Vendored
+30
View File
@@ -0,0 +1,30 @@
default: fmt vet errcheck test
# Taken from https://github.com/codecov/example-go#caveat-multiple-files
test:
echo "" > coverage.txt
for d in `go list ./... | grep -v vendor`; do \
go test -p 1 -v -timeout 90s -race -coverprofile=profile.out -covermode=atomic $$d || exit 1; \
if [ -f profile.out ]; then \
cat profile.out >> coverage.txt; \
rm profile.out; \
fi \
done
vet:
go vet ./...
# See https://github.com/kisielk/errcheck/pull/141 for details on ignorepkg
errcheck:
errcheck -ignorepkg fmt github.com/Shopify/sarama/...
fmt:
@if [ -n "$$(go fmt ./...)" ]; then echo 'Please run go fmt on your code.' && exit 1; fi
install_dependencies: install_errcheck get
install_errcheck:
go get github.com/kisielk/errcheck
get:
go get -t
vendor/github.com/Shopify/sarama/README.md
Generated Vendored
+39
View File
@@ -0,0 +1,39 @@
sarama
======
[![GoDoc](https://godoc.org/github.com/Shopify/sarama?status.png)](https://godoc.org/github.com/Shopify/sarama)
[![Build Status](https://travis-ci.org/Shopify/sarama.svg?branch=master)](https://travis-ci.org/Shopify/sarama)
[![Coverage](https://codecov.io/gh/Shopify/sarama/branch/master/graph/badge.svg)](https://codecov.io/gh/Shopify/sarama)
Sarama is an MIT-licensed Go client library for [Apache Kafka](https://kafka.apache.org/) version 0.8 (and later).
### Getting started
- API documentation and examples are available via [godoc](https://godoc.org/github.com/Shopify/sarama).
- Mocks for testing are available in the [mocks](./mocks) subpackage.
- The [examples](./examples) directory contains more elaborate example applications.
- The [tools](./tools) directory contains command line tools that can be useful for testing, diagnostics, and instrumentation.
You might also want to look at the [Frequently Asked Questions](https://github.com/Shopify/sarama/wiki/Frequently-Asked-Questions).
### Compatibility and API stability
Sarama provides a "2 releases + 2 months" compatibility guarantee: we support
the two latest stable releases of Kafka and Go, and we provide a two month
grace period for older releases. This means we currently officially support
Go 1.8 through 1.10, and Kafka 0.11 through 1.1, although older releases are
still likely to work.
Sarama follows semantic versioning and provides API stability via the gopkg.in service.
You can import a version with a guaranteed stable API via http://gopkg.in/Shopify/sarama.v1.
A changelog is available [here](CHANGELOG.md).
### Contributing
* Get started by checking our [contribution guidelines](https://github.com/Shopify/sarama/blob/master/.github/CONTRIBUTING.md).
* Read the [Sarama wiki](https://github.com/Shopify/sarama/wiki) for more
technical and design details.
* The [Kafka Protocol Specification](https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol)
contains a wealth of useful information.
* For more general issues, there is [a google group](https://groups.google.com/forum/#!forum/kafka-clients) for Kafka client developers.
* If you have any questions, just ask!
vendor/github.com/Shopify/sarama/Vagrantfile
Generated Vendored
+20
View File
@@ -0,0 +1,20 @@
# -*- mode: ruby -*-
# vi: set ft=ruby :
# Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
VAGRANTFILE_API_VERSION = "2"
# We have 5 * 192MB ZK processes and 5 * 320MB Kafka processes => 2560MB
MEMORY = 3072
Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
config.vm.box = "ubuntu/trusty64"
config.vm.provision :shell, path: "vagrant/provision.sh"
config.vm.network "private_network", ip: "192.168.100.67"
config.vm.provider "virtualbox" do |v|
v.memory = MEMORY
end
end
vendor/github.com/Shopify/sarama/acl_bindings.go
Generated Vendored
+119
View File
@@ -0,0 +1,119 @@
package sarama
type Resource struct {
ResourceType AclResourceType
ResourceName string
}
func (r *Resource) encode(pe packetEncoder) error {
pe.putInt8(int8(r.ResourceType))
if err := pe.putString(r.ResourceName); err != nil {
return err
}
return nil
}
func (r *Resource) decode(pd packetDecoder, version int16) (err error) {
resourceType, err := pd.getInt8()
if err != nil {
return err
}
r.ResourceType = AclResourceType(resourceType)
if r.ResourceName, err = pd.getString(); err != nil {
return err
}
return nil
}
type Acl struct {
Principal string
Host string
Operation AclOperation
PermissionType AclPermissionType
}
func (a *Acl) encode(pe packetEncoder) error {
if err := pe.putString(a.Principal); err != nil {
return err
}
if err := pe.putString(a.Host); err != nil {
return err
}
pe.putInt8(int8(a.Operation))
pe.putInt8(int8(a.PermissionType))
return nil
}
func (a *Acl) decode(pd packetDecoder, version int16) (err error) {
if a.Principal, err = pd.getString(); err != nil {
return err
}
if a.Host, err = pd.getString(); err != nil {
return err
}
operation, err := pd.getInt8()
if err != nil {
return err
}
a.Operation = AclOperation(operation)
permissionType, err := pd.getInt8()
if err != nil {
return err
}
a.PermissionType = AclPermissionType(permissionType)
return nil
}
type ResourceAcls struct {
Resource
Acls []*Acl
}
func (r *ResourceAcls) encode(pe packetEncoder) error {
if err := r.Resource.encode(pe); err != nil {
return err
}
if err := pe.putArrayLength(len(r.Acls)); err != nil {
return err
}
for _, acl := range r.Acls {
if err := acl.encode(pe); err != nil {
return err
}
}
return nil
}
func (r *ResourceAcls) decode(pd packetDecoder, version int16) error {
if err := r.Resource.decode(pd, version); err != nil {
return err
}
n, err := pd.getArrayLength()
if err != nil {
return err
}
r.Acls = make([]*Acl, n)
for i := 0; i < n; i++ {
r.Acls[i] = new(Acl)
if err := r.Acls[i].decode(pd, version); err != nil {
return err
}
}
return nil
}
vendor/github.com/Shopify/sarama/acl_create_request.go
Generated Vendored
+76
View File
@@ -0,0 +1,76 @@
package sarama
type CreateAclsRequest struct {
AclCreations []*AclCreation
}
func (c *CreateAclsRequest) encode(pe packetEncoder) error {
if err := pe.putArrayLength(len(c.AclCreations)); err != nil {
return err
}
for _, aclCreation := range c.AclCreations {
if err := aclCreation.encode(pe); err != nil {
return err
}
}
return nil
}
func (c *CreateAclsRequest) decode(pd packetDecoder, version int16) (err error) {
n, err := pd.getArrayLength()
if err != nil {
return err
}
c.AclCreations = make([]*AclCreation, n)
for i := 0; i < n; i++ {
c.AclCreations[i] = new(AclCreation)
if err := c.AclCreations[i].decode(pd, version); err != nil {
return err
}
}
return nil
}
func (d *CreateAclsRequest) key() int16 {
return 30
}
func (d *CreateAclsRequest) version() int16 {
return 0
}
func (d *CreateAclsRequest) requiredVersion() KafkaVersion {
return V0_11_0_0
}
type AclCreation struct {
Resource
Acl
}
func (a *AclCreation) encode(pe packetEncoder) error {
if err := a.Resource.encode(pe); err != nil {
return err
}
if err := a.Acl.encode(pe); err != nil {
return err
}
return nil
}
func (a *AclCreation) decode(pd packetDecoder, version int16) (err error) {
if err := a.Resource.decode(pd, version); err != nil {
return err
}
if err := a.Acl.decode(pd, version); err != nil {
return err
}
return nil
}
vendor/github.com/Shopify/sarama/acl_create_response.go
Generated Vendored
+88
View File
@@ -0,0 +1,88 @@
package sarama
import "time"
type CreateAclsResponse struct {
ThrottleTime time.Duration
AclCreationResponses []*AclCreationResponse
}
func (c *CreateAclsResponse) encode(pe packetEncoder) error {
pe.putInt32(int32(c.ThrottleTime / time.Millisecond))
if err := pe.putArrayLength(len(c.AclCreationResponses)); err != nil {
return err
}
for _, aclCreationResponse := range c.AclCreationResponses {
if err := aclCreationResponse.encode(pe); err != nil {
return err
}
}
return nil
}
func (c *CreateAclsResponse) decode(pd packetDecoder, version int16) (err error) {
throttleTime, err := pd.getInt32()
if err != nil {
return err
}
c.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
n, err := pd.getArrayLength()
if err != nil {
return err
}
c.AclCreationResponses = make([]*AclCreationResponse, n)
for i := 0; i < n; i++ {
c.AclCreationResponses[i] = new(AclCreationResponse)
if err := c.AclCreationResponses[i].decode(pd, version); err != nil {
return err
}
}
return nil
}
func (d *CreateAclsResponse) key() int16 {
return 30
}
func (d *CreateAclsResponse) version() int16 {
return 0
}
func (d *CreateAclsResponse) requiredVersion() KafkaVersion {
return V0_11_0_0
}
type AclCreationResponse struct {
Err KError
ErrMsg *string
}
func (a *AclCreationResponse) encode(pe packetEncoder) error {
pe.putInt16(int16(a.Err))
if err := pe.putNullableString(a.ErrMsg); err != nil {
return err
}
return nil
}
func (a *AclCreationResponse) decode(pd packetDecoder, version int16) (err error) {
kerr, err := pd.getInt16()
if err != nil {
return err
}
a.Err = KError(kerr)
if a.ErrMsg, err = pd.getNullableString(); err != nil {
return err
}
return nil
}
vendor/github.com/Shopify/sarama/acl_delete_request.go
Generated Vendored
+48
View File
@@ -0,0 +1,48 @@
package sarama
type DeleteAclsRequest struct {
Filters []*AclFilter
}
func (d *DeleteAclsRequest) encode(pe packetEncoder) error {
if err := pe.putArrayLength(len(d.Filters)); err != nil {
return err
}
for _, filter := range d.Filters {
if err := filter.encode(pe); err != nil {
return err
}
}
return nil
}
func (d *DeleteAclsRequest) decode(pd packetDecoder, version int16) (err error) {
n, err := pd.getArrayLength()
if err != nil {
return err
}
d.Filters = make([]*AclFilter, n)
for i := 0; i < n; i++ {
d.Filters[i] = new(AclFilter)
if err := d.Filters[i].decode(pd, version); err != nil {
return err
}
}
return nil
}
func (d *DeleteAclsRequest) key() int16 {
return 31
}
func (d *DeleteAclsRequest) version() int16 {
return 0
}
func (d *DeleteAclsRequest) requiredVersion() KafkaVersion {
return V0_11_0_0
}
vendor/github.com/Shopify/sarama/acl_delete_response.go
Generated Vendored
+155
View File
@@ -0,0 +1,155 @@
package sarama
import "time"
type DeleteAclsResponse struct {
ThrottleTime time.Duration
FilterResponses []*FilterResponse
}
func (a *DeleteAclsResponse) encode(pe packetEncoder) error {
pe.putInt32(int32(a.ThrottleTime / time.Millisecond))
if err := pe.putArrayLength(len(a.FilterResponses)); err != nil {
return err
}
for _, filterResponse := range a.FilterResponses {
if err := filterResponse.encode(pe); err != nil {
return err
}
}
return nil
}
func (a *DeleteAclsResponse) decode(pd packetDecoder, version int16) (err error) {
throttleTime, err := pd.getInt32()
if err != nil {
return err
}
a.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
n, err := pd.getArrayLength()
if err != nil {
return err
}
a.FilterResponses = make([]*FilterResponse, n)
for i := 0; i < n; i++ {
a.FilterResponses[i] = new(FilterResponse)
if err := a.FilterResponses[i].decode(pd, version); err != nil {
return err
}
}
return nil
}
func (d *DeleteAclsResponse) key() int16 {
return 31
}
func (d *DeleteAclsResponse) version() int16 {
return 0
}
func (d *DeleteAclsResponse) requiredVersion() KafkaVersion {
return V0_11_0_0
}
type FilterResponse struct {
Err KError
ErrMsg *string
MatchingAcls []*MatchingAcl
}
func (f *FilterResponse) encode(pe packetEncoder) error {
pe.putInt16(int16(f.Err))
if err := pe.putNullableString(f.ErrMsg); err != nil {
return err
}
if err := pe.putArrayLength(len(f.MatchingAcls)); err != nil {
return err
}
for _, matchingAcl := range f.MatchingAcls {
if err := matchingAcl.encode(pe); err != nil {
return err
}
}
return nil
}
func (f *FilterResponse) decode(pd packetDecoder, version int16) (err error) {
kerr, err := pd.getInt16()
if err != nil {
return err
}
f.Err = KError(kerr)
if f.ErrMsg, err = pd.getNullableString(); err != nil {
return err
}
n, err := pd.getArrayLength()
if err != nil {
return err
}
f.MatchingAcls = make([]*MatchingAcl, n)
for i := 0; i < n; i++ {
f.MatchingAcls[i] = new(MatchingAcl)
if err := f.MatchingAcls[i].decode(pd, version); err != nil {
return err
}
}
return nil
}
type MatchingAcl struct {
Err KError
ErrMsg *string
Resource
Acl
}
func (m *MatchingAcl) encode(pe packetEncoder) error {
pe.putInt16(int16(m.Err))
if err := pe.putNullableString(m.ErrMsg); err != nil {
return err
}
if err := m.Resource.encode(pe); err != nil {
return err
}
if err := m.Acl.encode(pe); err != nil {
return err
}
return nil
}
func (m *MatchingAcl) decode(pd packetDecoder, version int16) (err error) {
kerr, err := pd.getInt16()
if err != nil {
return err
}
m.Err = KError(kerr)
if m.ErrMsg, err = pd.getNullableString(); err != nil {
return err
}
if err := m.Resource.decode(pd, version); err != nil {
return err
}
if err := m.Acl.decode(pd, version); err != nil {
return err
}
return nil
}
vendor/github.com/Shopify/sarama/acl_describe_request.go
Generated Vendored
+25
View File
@@ -0,0 +1,25 @@
package sarama
type DescribeAclsRequest struct {
AclFilter
}
func (d *DescribeAclsRequest) encode(pe packetEncoder) error {
return d.AclFilter.encode(pe)
}
func (d *DescribeAclsRequest) decode(pd packetDecoder, version int16) (err error) {
return d.AclFilter.decode(pd, version)
}
func (d *DescribeAclsRequest) key() int16 {
return 29
}
func (d *DescribeAclsRequest) version() int16 {
return 0
}
func (d *DescribeAclsRequest) requiredVersion() KafkaVersion {
return V0_11_0_0
}
vendor/github.com/Shopify/sarama/acl_describe_response.go
Generated Vendored
+80
View File
@@ -0,0 +1,80 @@
package sarama
import "time"
type DescribeAclsResponse struct {
ThrottleTime time.Duration
Err KError
ErrMsg *string
ResourceAcls []*ResourceAcls
}
func (d *DescribeAclsResponse) encode(pe packetEncoder) error {
pe.putInt32(int32(d.ThrottleTime / time.Millisecond))
pe.putInt16(int16(d.Err))
if err := pe.putNullableString(d.ErrMsg); err != nil {
return err
}
if err := pe.putArrayLength(len(d.ResourceAcls)); err != nil {
return err
}
for _, resourceAcl := range d.ResourceAcls {
if err := resourceAcl.encode(pe); err != nil {
return err
}
}
return nil
}
func (d *DescribeAclsResponse) decode(pd packetDecoder, version int16) (err error) {
throttleTime, err := pd.getInt32()
if err != nil {
return err
}
d.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
kerr, err := pd.getInt16()
if err != nil {
return err
}
d.Err = KError(kerr)
errmsg, err := pd.getString()
if err != nil {
return err
}
if errmsg != "" {
d.ErrMsg = &errmsg
}
n, err := pd.getArrayLength()
if err != nil {
return err
}
d.ResourceAcls = make([]*ResourceAcls, n)
for i := 0; i < n; i++ {
d.ResourceAcls[i] = new(ResourceAcls)
if err := d.ResourceAcls[i].decode(pd, version); err != nil {
return err
}
}
return nil
}
func (d *DescribeAclsResponse) key() int16 {
return 29
}
func (d *DescribeAclsResponse) version() int16 {
return 0
}
func (d *DescribeAclsResponse) requiredVersion() KafkaVersion {
return V0_11_0_0
}
vendor/github.com/Shopify/sarama/acl_filter.go
Generated Vendored
+61
View File
@@ -0,0 +1,61 @@
package sarama
type AclFilter struct {
ResourceType AclResourceType
ResourceName *string
Principal *string
Host *string
Operation AclOperation
PermissionType AclPermissionType
}
func (a *AclFilter) encode(pe packetEncoder) error {
pe.putInt8(int8(a.ResourceType))
if err := pe.putNullableString(a.ResourceName); err != nil {
return err
}
if err := pe.putNullableString(a.Principal); err != nil {
return err
}
if err := pe.putNullableString(a.Host); err != nil {
return err
}
pe.putInt8(int8(a.Operation))
pe.putInt8(int8(a.PermissionType))
return nil
}
func (a *AclFilter) decode(pd packetDecoder, version int16) (err error) {
resourceType, err := pd.getInt8()
if err != nil {
return err
}
a.ResourceType = AclResourceType(resourceType)
if a.ResourceName, err = pd.getNullableString(); err != nil {
return err
}
if a.Principal, err = pd.getNullableString(); err != nil {
return err
}
if a.Host, err = pd.getNullableString(); err != nil {
return err
}
operation, err := pd.getInt8()
if err != nil {
return err
}
a.Operation = AclOperation(operation)
permissionType, err := pd.getInt8()
if err != nil {
return err
}
a.PermissionType = AclPermissionType(permissionType)
return nil
}
vendor/github.com/Shopify/sarama/acl_types.go
Generated Vendored
+42
View File
@@ -0,0 +1,42 @@
package sarama
type AclOperation int
// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/acl/AclOperation.java
const (
AclOperationUnknown AclOperation = 0
AclOperationAny AclOperation = 1
AclOperationAll AclOperation = 2
AclOperationRead AclOperation = 3
AclOperationWrite AclOperation = 4
AclOperationCreate AclOperation = 5
AclOperationDelete AclOperation = 6
AclOperationAlter AclOperation = 7
AclOperationDescribe AclOperation = 8
AclOperationClusterAction AclOperation = 9
AclOperationDescribeConfigs AclOperation = 10
AclOperationAlterConfigs AclOperation = 11
AclOperationIdempotentWrite AclOperation = 12
)
type AclPermissionType int
// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/acl/AclPermissionType.java
const (
AclPermissionUnknown AclPermissionType = 0
AclPermissionAny AclPermissionType = 1
AclPermissionDeny AclPermissionType = 2
AclPermissionAllow AclPermissionType = 3
)
type AclResourceType int
// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/resource/ResourceType.java
const (
AclResourceUnknown AclResourceType = 0
AclResourceAny AclResourceType = 1
AclResourceTopic AclResourceType = 2
AclResourceGroup AclResourceType = 3
AclResourceCluster AclResourceType = 4
AclResourceTransactionalID AclResourceType = 5
)

Some files were not shown because too many files have changed in this diff Show More