diff --git a/internal/types/evaluation.go b/internal/types/evaluation.go index 3e8d1d07f..1ffb04b26 100644 --- a/internal/types/evaluation.go +++ b/internal/types/evaluation.go @@ -2,13 +2,30 @@ package types import ( "encoding/json" + "os" + "path/filepath" "time" "github.com/yanyiwu/gojieba" ) // Jieba is a global instance of Chinese text segmentation tool -var Jieba *gojieba.Jieba = gojieba.NewJieba() +var Jieba *gojieba.Jieba = newJieba() + +func newJieba() *gojieba.Jieba { + dictDir := os.Getenv("JIEBA_DICT_DIR") + if dictDir == "" { + return gojieba.NewJieba() + } + + return gojieba.NewJieba( + filepath.Join(dictDir, "jieba.dict.utf8"), + filepath.Join(dictDir, "hmm_model.utf8"), + filepath.Join(dictDir, "user.dict.utf8"), + filepath.Join(dictDir, "idf.utf8"), + filepath.Join(dictDir, "stop_words.utf8"), + ) +} // EvaluationStatue represents the status of an evaluation task type EvaluationStatue int