Merge pull request #9664 from yousong/automated-cherry-pick-of-#9625-upstream-release-3.7

Automated cherry pick of #9625: Feature/yousong y18n check

.circleci/config.yml

@@ -5,10 +5,10 @@ version: 2
 jobs:
   check:
     docker:
-      - image: yunionio/centos-build:1.0-1
+      - image: yunionio/centos-build:1.1-4
         environment:
           ONECLOUD_CI_BUILD: "1"
-    working_directory: /root/go/onecloud
+    working_directory: /root/go/src/yunion.io/x/onecloud
     steps:
       - checkout
       - run:
@@ -18,7 +18,7 @@ jobs:
     requires:
       - check
     docker:
-      - image: yunionio/centos-build:1.0-1
+      - image: yunionio/centos-build:1.1-4
         environment:
           ONECLOUD_CI_BUILD: "1"
 
@@ -26,7 +26,7 @@ jobs:
     #### expecting it in the form of
     ####   /go/src/github.com/circleci/go-tool
     ####   /go/src/bitbucket.org/circleci/go-tool
-    working_directory: /root/go/onecloud
+    working_directory: /root/go/src/yunion.io/x/onecloud
     parallelism: 4
     steps:
       - checkout
@@ -51,10 +51,10 @@ jobs:
     requires:
       - check
     docker:
-      - image: yunionio/centos-build:1.0-1
+      - image: yunionio/centos-build:1.1-4
         environment:
-          GO111MODULE: "on"
-    working_directory: /root/go/onecloud
+          ONECLOUD_CI_BUILD: "1"
+    working_directory: /root/go/src/yunion.io/x/onecloud
     parallelism: 4
     steps:
       - checkout

Makefile

@@ -173,10 +173,52 @@ vet-check:
 	./scripts/vet.sh chk
 .PHONY: vet-check
 
+comma:=,
+space:=$(space) $(space)
+# NOTE: keep y18n-packages in alphabetical order
+y18n-src-lang := en-US
+y18n-lang     := en-US,zh-CN
+y18n-packages := \
+		yunion.io/x/onecloud/cmd/keystone \
+		yunion.io/x/onecloud/cmd/monitor \
+		yunion.io/x/onecloud/cmd/region \
+		yunion.io/x/onecloud/cmd/yunionconf \
+
+define y18n-gen
+	set -o errexit; \
+	set -o pipefail; \
+	export GO111MODULE=off; \
+	y18n \
+		-chdir $(CURDIR) \
+		-dir ./locales/ \
+		-out ./locales/locales.go \
+		-lang $(y18n-lang) \
+		$(y18n-packages) \
+		; \
+	$(foreach lang,$(filter-out $(y18n-src-lang),$(subst $(comma), ,$(y18n-lang))),cp ./locales/$(lang)/{out,messages}.gotext.json;) \
+
+endef
+
+y18n-gen:
+	$(y18n-gen)
+	$(y18n-gen)
+
+.PHONY: y18n-gen
+
+y18n-check:
+	$(y18n-gen)
+	if git status --short ./locales | sed 's/^/$@: /' | grep .; then \
+		echo "$@: Locales content needs care" >&2 ; \
+		false; \
+	fi
+
+.PHONY: y18n-check
+
 check: fmt-check
 check: gendocgo-check
 check: goimports-check
 check: vet-check
+#check: y18n-check
 .PHONY: check
 
 
@@ -203,70 +245,6 @@ mod:
 	go mod tidy
 	go mod vendor -v
 
-
-DOCKER_CENTOS_BUILD_IMAGE?=registry.cn-beijing.aliyuncs.com/yunionio/centos-build:1.1-3
-
-define dockerCentOSBuildCmd
-set -o xtrace
-set -o errexit
-set -o pipefail
-cd /root/onecloud
-export GOFLAGS=-mod=vendor
-make $(1)
-chown -R $(shell id -u):$(shell id -g) _output
-endef
-
-docker-centos-build: export dockerCentOSBuildCmd:=$(call dockerCentOSBuildCmd,$(F))
-docker-centos-build:
-	docker rm --force onecloud-ci-build &>/dev/null || true
-	docker run \
-		--name onecloud-docker-centos-build \
-		--rm \
-		--volume $(CURDIR):/root/onecloud \
-		--volume $(CURDIR)/_output/_cache:/root/.cache \
-		$(DOCKER_CENTOS_BUILD_IMAGE) \
-		/bin/bash -c "$$dockerCentOSBuildCmd"
-	chown -R $$(id -u):$$(id -g) _output
-	ls -lh _output/bin
-
-# NOTE we need a way to stop and remove the container started by docker-build.
-# No --tty, --stop-signal won't work
-docker-centos-build-stop:
-	docker stop --time 0 onecloud-docker-centos-build || true
-
-.PHONY: docker-centos-build
-.PHONY: docker-centos-build-stop
-
-DOCKER_ALPINE_BUILD_IMAGE?=registry.cn-beijing.aliyuncs.com/yunionio/alpine-build:1.0-5
-
-define dockerAlpineBuildCmd
-set -o xtrace
-set -o errexit
-set -o pipefail
-cd /root/go/src/yunion.io/x/onecloud
-export GOFLAGS=-mod=vendor
-make $(1)
-chown -R $(shell id -u):$(shell id -g) _output
-endef
-
-docker-alpine-build: export dockerAlpineBuildCmd:=$(call dockerAlpineBuildCmd,$(F))
-docker-alpine-build:
-	docker rm --force onecloud-docker-alpine-build &>/dev/null || true
-	docker run --rm \
-		--name onecloud-docker-alpine-build \
-		-v $(CURDIR):/root/go/src/yunion.io/x/onecloud \
-		-v $(CURDIR)/_output/alpine-build:/root/go/src/yunion.io/x/onecloud/_output \
-		-v $(CURDIR)/_output/alpine-build/_cache:/root/.cache \
-		$(DOCKER_ALPINE_BUILD_IMAGE) \
-		/bin/sh -c "$$dockerAlpineBuildCmd"
-	ls -lh _output/alpine-build/bin
-
-docker-alpine-build-stop:
-	docker stop --time 0 onecloud-docker-alpine-build || true
-
-.PHONY: docker-alpine-build
-.PHONY: docker-alpine-build-stop
-
 define helpText
 Build with docker
 
@@ -326,3 +304,6 @@ image:
 
 %:
 	@:
+
+ModName:=yunion.io/x/onecloud
+include $(CURDIR)/Makefile.common.mk

Makefile.common.mk

@@ -0,0 +1,83 @@
+ifeq ($(__inc_Makefile_common_mk),)
+__inc_Makefile_common_mk:=1
+
+ifeq ($(ModName),)
+  $(error ModName must be set, e.g. yunion.io/x/onecloud)
+endif
+ModBaseName:=$(notdir $(ModName))
+
+DockerImageRegistry?=registry.cn-beijing.aliyuncs.com
+DockerImageAlpineBuild?=$(DockerImageRegistry)/yunionio/alpine-build:1.0-5
+DockerImageCentOSBuild?=$(DockerImageRegistry)/yunionio/centos-build:1.1-4
+
+
+EnvIf=$(if $($(1)),$(1)=$($(1)))
+
+define dockerCentOSBuildCmd
+set -o xtrace
+set -o errexit
+set -o pipefail
+cd /root/go/src/yunion.io/x/$(ModBaseName)
+env \
+	$(call EnvIf,GOARCH) \
+	$(call EnvIf,GOOS) \
+	$(call EnvIf,CGO_ENABLED) \
+	make $(1)
+chown -R $(shell id -u):$(shell id -g) _output
+endef
+
+docker-centos-build: export dockerCentOSBuildCmd:=$(call dockerCentOSBuildCmd,$(F))
+docker-centos-build:
+	docker rm --force docker-centos-build-$(ModBaseName) &>/dev/null || true
+	docker run \
+		--rm \
+		--name docker-centos-build-$(ModBaseName) \
+		-v $(CURDIR):/root/go/src/yunion.io/x/$(ModBaseName) \
+		-v $(CURDIR)/_output/centos-build:/root/go/src/yunion.io/x/$(ModBaseName)/_output \
+		-v $(CURDIR)/_output/centos-build/_cache:/root/.cache \
+		$(DockerImageCentOSBuild) \
+		/bin/bash -c "$$dockerCentOSBuildCmd"
+	ls -lh _output/centos-build/bin
+
+# NOTE we need a way to stop and remove the container started by docker-build.
+# No --tty, --stop-signal won't work
+docker-centos-build-stop:
+	docker stop --time 0 docker-centos-build-$(ModBaseName) || true
+
+.PHONY: docker-centos-build
+.PHONY: docker-centos-build-stop
+
+
+define dockerAlpineBuildCmd
+set -o xtrace
+set -o errexit
+set -o pipefail
+cd /root/go/src/yunion.io/x/$(ModBaseName)
+env \
+	$(call EnvIf,GOARCH) \
+	$(call EnvIf,GOOS) \
+	$(call EnvIf,CGO_ENABLED) \
+	make $(1)
+chown -R $(shell id -u):$(shell id -g) _output
+endef
+
+docker-alpine-build: export dockerAlpineBuildCmd:=$(call dockerAlpineBuildCmd,$(F))
+docker-alpine-build:
+	docker rm --force docker-alpine-build-$(ModBaseName) &>/dev/null || true
+	docker run \
+		--rm \
+		--name docker-alpine-build-$(ModBaseName) \
+		-v $(CURDIR):/root/go/src/yunion.io/x/$(ModBaseName) \
+		-v $(CURDIR)/_output/alpine-build:/root/go/src/yunion.io/x/$(ModBaseName)/_output \
+		-v $(CURDIR)/_output/alpine-build/_cache:/root/.cache \
+		$(DockerImageAlpineBuild) \
+		/bin/sh -c "$$dockerAlpineBuildCmd"
+	ls -lh _output/alpine-build/bin
+
+docker-alpine-build-stop:
+	docker stop --time 0 docker-alpine-build-$(ModBaseName) || true
+
+.PHONY: docker-alpine-build
+.PHONY: docker-alpine-build-stop
+
+endif # __inc_Makefile_common_mk

build/docker/Dockerfile.host-deployer

@@ -6,4 +6,4 @@ ENV TZ Asia/Shanghai
 
 RUN mkdir -p /opt/yunion/bin
 
-ADD ./_output/alpine-build/bin/host-deployer /opt/yunion/bin/host-deployer
+ADD ./_output/centos-build/bin/host-deployer /opt/yunion/bin/host-deployer

pkg/apis/compute/cloudaccount_const.go

@@ -17,6 +17,7 @@ package compute
 import (
 	"yunion.io/x/onecloud/pkg/apis"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/util/choices"
 )
 
 const (
@@ -67,6 +68,12 @@ const (
 	CLOUD_ACCOUNT_WIRE_LEVEL_CLUSTER    = "cluster"
 )
 
+var CLOUD_ACCOUNT_WIRE_LEVELS = choices.NewChoices(
+	CLOUD_ACCOUNT_WIRE_LEVEL_VCENTER,
+	CLOUD_ACCOUNT_WIRE_LEVEL_DATACENTER,
+	CLOUD_ACCOUNT_WIRE_LEVEL_CLUSTER,
+)
+
 const (
 	CLOUD_ACCESS_ENV_AWS_GLOBAL          = CLOUD_PROVIDER_AWS + "-int"
 	CLOUD_ACCESS_ENV_AWS_CHINA           = CLOUD_PROVIDER_AWS

pkg/compute/models/cloudaccounts.go

@@ -417,7 +417,7 @@ func (scm *SCloudaccountManager) hostVMIPsPrepareNets(ctx context.Context, clien
 			}
 		}
 	default:
-		return nil, httperrors.NewInputParameterError("valid wire_level_for_vmware, accept vcenter, datacenter, cluster")
+		return nil, httperrors.NewInputParameterError("invalid wire_level_for_vmware %q, accept %s", wireLevel, api.CLOUD_ACCOUNT_WIRE_LEVELS)
 	}
 	return ret, nil
 }

scripts/coverage.sh

@@ -30,8 +30,14 @@ function push_to_codecov() {
 covermode=${COVERMODE:-atomic}
 coverdir=$(mktemp -d /tmp/coverage.XXXXXXXXXX)
 profile="${coverdir}/profile.out"
+
+cwd="$PWD"
+if [ -s "$cwd/go.mod" -a -d "$cwd/vendor" ]; then
+	mod_args=(-mod vendor)
+fi
+
 if [ -z "$pkgs" ]; then
-	pkgs="$(go list -mod vendor -test ./... | grep '\.test$' | sed -e 's/\.test$//')"
+	pkgs="$(go list "${mod_args[@]}" -test ./... | grep '\.test$' | sed -e 's/\.test$//')"
 	pkgs="$(echo "$pkgs" | grep -vE 'host-image|hostimage')"
 fi
 if type circleci &>/dev/null; then
@@ -44,7 +50,7 @@ echo "$pkgs" | xargs -n 8 --no-run-if-empty echo \
 		go test -v \
 			-coverprofile="$profile.tmp" \
 			-covermode="$covermode" \
-			-mod vendor \
+			"${mod_args[@]}" \
 			-ldflags '-w' \
 			$batch; \
 		tail -n +2 "$profile.tmp" >>"$profile"; \

scripts/docker_push.sh

@@ -1,4 +1,6 @@
 #!/bin/bash
+#
+# vi: expandtab tabstop=4 shiftwidth=0
 
 set -o errexit
 set -o pipefail
@@ -45,31 +47,18 @@ build_bin() {
     local BUILD_CGO=$3
     case "$1" in
         baremetal-agent)
+            rm -vf _output/bin/$1
+            rm -rvf _output/bin/bundles/$1
             GOOS=linux make cmd/$1
             ;;
         climc)
-            docker run --rm \
-                -v $SRC_DIR:/root/go/src/yunion.io/x/onecloud \
-                -v $SRC_DIR/_output/alpine-build:/root/go/src/yunion.io/x/onecloud/_output \
-                -v $SRC_DIR/_output/alpine-build/_cache:/root/.cache \
-                registry.cn-beijing.aliyuncs.com/yunionio/alpine-build:1.0-5 \
-                /bin/sh -c "set -ex; cd /root/go/src/yunion.io/x/onecloud; $BUILD_ARCH $BUILD_CGO GOOS=linux make cmd/$1 cmd/*cli; chown -R $(id -u):$(id -g) _output"
+            env $BUILD_ARCH $BUILD_CGO make -C "$SRC_DIR" docker-alpine-build F="cmd/$1 cmd/*cli"
             ;;
         host-deployer)
-            docker run --rm \
-                -v $SRC_DIR:/root/go/src/yunion.io/x/onecloud \
-                -v $SRC_DIR/_output/alpine-build:/root/go/src/yunion.io/x/onecloud/_output \
-                -v $SRC_DIR/_output/alpine-build/_cache:/root/.cache \
-                registry.cn-beijing.aliyuncs.com/yunionio/centos-build:1.1-3 \
-                /bin/sh -c "set -ex; cd /root/go/src/yunion.io/x/onecloud; $BUILD_ARCH $BUILD_CGO GOOS=linux make cmd/$1; chown -R $(id -u):$(id -g) _output"
+            env $BUILD_ARCH $BUILD_CGO make -C "$SRC_DIR" docker-centos-build F="cmd/$1"
             ;;
         *)
-            docker run --rm \
-                -v $SRC_DIR:/root/go/src/yunion.io/x/onecloud \
-                -v $SRC_DIR/_output/alpine-build:/root/go/src/yunion.io/x/onecloud/_output \
-                -v $SRC_DIR/_output/alpine-build/_cache:/root/.cache \
-                registry.cn-beijing.aliyuncs.com/yunionio/alpine-build:1.0-5 \
-                /bin/sh -c "set -ex; cd /root/go/src/yunion.io/x/onecloud; $BUILD_ARCH $BUILD_CGO GOOS=linux make cmd/$1; chown -R $(id -u):$(id -g) _output"
+            env $BUILD_ARCH $BUILD_CGO make -C "$SRC_DIR" docker-alpine-build F="cmd/$1"
             ;;
     esac
 }
@@ -88,7 +77,7 @@ build_image() {
     local tag=$1
     local file=$2
     local path=$3
-    docker build -t "$tag" -f "$2" "$3"
+    docker buildx build -t "$tag" -f "$2" "$3"
 }
 
 buildx_and_push() {
@@ -204,7 +193,9 @@ for component in $COMPONENTS; do
             done
             ;;
         *)
-            general_build $component $ARCH
+            if [ -e "$DOCKER_DIR/Dockerfile.$component" ]; then
+                general_build $component $ARCH
+            fi
             ;;
     esac
 done