命令记不住

登陆

Golang 接入 Elasticsearch

  |   129 浏览

Golang接入Elasticsearch

1.导入数据的设计

  • 设计结构体，哪些字段参与搜索，哪些字段需要返回
  • 导入数据，获得默认mapping
  • 修改mapping，需要重建索引
  • 为中文分词设置mapping

2.WebDemo引入es

2.1 安装es go package
go get github.com/olivere/elastic
2.2 文章数据的代码

分析models/article_model.go，找到访问文章数据的代码QueryArticlesWithCon

写代码，访问article全表

package main

import (
	"blogweb_gin/database"
	"fmt"
	"grpcdemo/models"
)

func ArticleGetAll() {
	database.InitMysql()
	list, err := models.QueryArticlesWithCon("")
	if err == nil {
		for _, item := range list {
			fmt.Printf("%d, %s\n", item.Id, item.Content)
		}
	} else {
		fmt.Println(err.Error())
	}
}

func main() {
	ArticleGetAll()
}

运行代码，看到目前数据库有两条记录

PS D:\mygo\src\grpcdemo\es> .\es.exe
InitMysql....
select id,title,tags,short,content,author,createtime from article
2, 理解Elastic Search
3, etcd学习

3.mysql数据导入ES

3.1es的convert方法

编写搜索工具包，定义搜索文档实体，写convert方法

grpcdemo-es\models\esutils.go

package models

import (
	"context"
	"fmt"

	"github.com/olivere/elastic"
)

type ArticleSE struct {
	Id         int    `json:"id"`
	Title      string `json:"title"`
	Tags       string `json:"tags"`
	Short      string `json:"short"`
	Content    string `json:"content"`
	Author     string `json:"author"`
	Createtime int64  `json:"create_time"`
}

func (o *Article) ToSearchEntity() *ArticleSE {
	return &ArticleSE{
		Id:         o.Id,
		Title:      o.Title,
		Tags:       o.Tags,
		Short:      o.Short,
		Content:    o.Content,
		Author:     o.Author,
		Createtime: o.Createtime,
	}
}

// 写单个记录
func SearchArticleBuildOne(ctx context.Context, item *Article, client *elastic.Client) {
	if item != nil {
		fmt.Printf("SearchArticleBuildOne: %d", item.Id)
		se_index := "blogweb_article"
		se_type := "doc"
		se_id := fmt.Sprintf("%d", item.Id)
		e := item.ToSearchEntity()

		put1, err := client.Index().
			Index(se_index).
			Type(se_type).
			Id(se_id).
			BodyJson(e).
			Do(ctx)
		if err != nil {
			// Handle error
			fmt.Println(err.Error())
		}
		fmt.Printf("Indexed %s to index %s, type %s\n", put1.Id, put1.Index, put1.Type)
	}
}
3.2 main引入es包

重写main.go，引入es包

package main

import (
	"blogweb_gin/database"
	"context"
	"fmt"
	"grpcdemo/models"

	"github.com/olivere/elastic"
)

// ArticleGetAll
func ArticleGetAll() {

	database.InitMysql()
	list, err := models.QueryArticlesWithCon("")
	if err == nil {
		ctx := context.Background()
		client, err := elastic.NewSimpleClient(elastic.SetURL("http://127.0.0.1:9200/"))
		if err != nil {
			// Handle error
			fmt.Println(err.Error())
			return
		}
		defer client.Stop()
		for _, item := range list {
			fmt.Printf("article: %d, %s\n", item.Id, item.Content)
			models.SearchArticleBuildOne(ctx, &item, client)
		}
	} else {
		fmt.Println(err.Error())
	}
}

func main() {
	ArticleGetAll()
}

运行代码

PS D:\mygo\src\grpcdemo\es> .\es.exe
InitMysql....
select id,title,tags,short,content,author,createtime from article
article: 2, 理解Elastic Search
SearchArticleBuildOne: 2SearchArticleBuildOne: 2Indexed 2 to index blogweb_article, type doc
article: 3, etcd学习
SearchArticleBuildOne: 3SearchArticleBuildOne: 3Indexed 3 to index blogweb_article, type doc
3.3 查看索引

查看索引列表 http://localhost:9200/_cat/indices

yellow open blogweb_article           L8CqHvLYRuOnkWyFSJZGpA 5 1  2 0 12.9kb 12.9kb

查询搜索记录

GET /blogweb_article/doc/_search


{
  "took" : 1,
  "timed_out" : false,
  "_shards" : {
    "total" : 5,
    "successful" : 5,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : 2,
    "max_score" : 1.0,
    "hits" : [
      {
        "_index" : "blogweb_article",
        "_type" : "doc",
        "_id" : "2",
        "_score" : 1.0,
        "_source" : {
          "id" : 2,
          "title" : "another one",
          "tags" : "hello1",
          "short" : "world",
          "content" : "理解Elastic Search",
          "author" : "",
          "create_time" : 1584860168
        }
      },
      {
        "_index" : "blogweb_article",
        "_type" : "doc",
        "_id" : "3",
        "_score" : 1.0,
        "_source" : {
          "id" : 3,
          "title" : "新文章",
          "tags" : "go",
          "short" : "学习golang",
          "content" : "etcd学习",
          "author" : "",
          "create_time" : 1585467683
        }
      }
    ]
  }
}

简单搜索

GET /blogweb_article/doc/_search
{
    "query" : {
        "match" : {
            "content": "理"
        }
    }
}

因为不理解中文分词，“理”和“解”被拆分，能够搜到结果

4.设置mapping

4.1 查看默认mapping
GET /blogweb_article/_mapping

{
  "blogweb_article" : {
    "mappings" : {
      "doc" : {
        "properties" : {
          "author" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          },
          "content" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          },
          "create_time" : {
            "type" : "long"
          },
          "id" : {
            "type" : "long"
          },
          "short" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          },
          "tags" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          },
          "title" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          }
        }
      }
    }
  }
}
4.2 定制字段类型
DELETE /blogweb_article
PUT /blogweb_article
{
	"settings": {
        "number_of_shards" :   1,
        "number_of_replicas" : 0
    },
    "mappings" : {
      "doc" : {
        "properties" : {
          "author" : {
            "type" : "text",
            "analyzer" : "ik_max_word"
          },
          "content" : {
            "type" : "text",
            "analyzer" : "ik_max_word"
          },
          "create_time" : {
            "type" : "long"
          },
          "id" : {
            "type" : "long"
          },
          "short" : {
            "type" : "text",
            "analyzer" : "ik_max_word"
          },
          "tags" : {
            "type" : "text",
            "analyzer" : "ik_max_word"
          },
          "title" : {
            "type" : "text",
            "analyzer" : "ik_max_word"
          }
        }
      }
    }
}
4.3 导入数据再搜索

再次导入数据

再次搜索

GET /blogweb_article/doc/_search
{
    "query" : {
        "match" : {
            "content": "理"
        }
    }
}

没有结果，搜“理解”，有结果

{
  "took" : 1,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : 1,
    "max_score" : 0.64072424,
    "hits" : [
      {
        "_index" : "blogweb_article",
        "_type" : "doc",
        "_id" : "2",
        "_score" : 0.64072424,
        "_source" : {
          "id" : 2,
          "title" : "another one",
          "tags" : "hello1",
          "short" : "world",
          "content" : "理解Elastic Search",
          "author" : "",
          "create_time" : 1584860168
        }
      }
    ]
  }
}

5.查询操作

  • 删除文档
  • 构造搜索条件
  • 获取搜索结果

补充工具类，完成删除方法和查询方法

package models

import (
	"context"
	"encoding/json"
	"fmt"
	"reflect"

	"github.com/olivere/elastic"
)

type ArticleSE struct {
	Id         int    `json:"id"`
	Title      string `json:"title"`
	Tags       string `json:"tags"`
	Short      string `json:"short"`
	Content    string `json:"content"`
	Author     string `json:"author"`
	Createtime int64  `json:"create_time"`
}

func (o *Article) ToSearchEntity() *ArticleSE {
	return &ArticleSE{
		Id:         o.Id,
		Title:      o.Title,
		Tags:       o.Tags,
		Short:      o.Short,
		Content:    o.Content,
		Author:     o.Author,
		Createtime: o.Createtime,
	}
}

func ArticleFromSearchEntity(value *ArticleSE) *Article {
	data := &Article{
		Id:         value.Id,
		Title:      value.Title,
		Tags:       value.Tags,
		Short:      value.Short,
		Content:    value.Content,
		Author:     value.Author,
		Createtime: value.Createtime,
	}
	return data
}

// 搜索相关
func SearchArticleBuildOne(ctx context.Context, item *Article, client *elastic.Client) {
	fmt.Printf("SearchArticleBuildOne: %d", item.Id)
	if item != nil {
		fmt.Printf("SearchArticleBuildOne: %d", item.Id)
		se_index := "blogweb_article"
		se_type := "doc"
		se_id := fmt.Sprintf("%d", item.Id)
		e := item.ToSearchEntity()

		put1, err := client.Index().
			Index(se_index).
			Type(se_type).
			Id(se_id).
			BodyJson(e).
			Do(ctx)
		if err != nil {
			// Handle error
			fmt.Println(err.Error())
		}
		fmt.Printf("Indexed %s to index %s, type %s\n", put1.Id, put1.Index, put1.Type)
	}
}

func SearchArticleRemoveOne(id int64) {
	fmt.Printf("SearchArticleRemoveOne: %d", id)
	client, err := elastic.NewSimpleClient(elastic.SetURL("http://127.0.0.1:9200/"))
	if err != nil {
		// Handle error
		fmt.Println(err.Error())
		return
	}
	defer client.Stop()

	ctx := context.Background()
	se_index := "blogweb_article"
	se_type := "doc"
	se_id := fmt.Sprintf("%d", id)
	res, err := client.Delete().
		Index(se_index).
		Type(se_type).
		Id(se_id).
		Do(ctx)
	if err != nil {
		// Handle error
		fmt.Println(err.Error())
	} else {
		fmt.Printf("delete result: %d, %s", res.Status, res.Result)
	}
}

func DumpQuery(src interface{}, err error) {
	if err != nil {
		panic(err)
	}
	data, err := json.MarshalIndent(src, "", "  ")
	if err != nil {
		panic(err)
	}
	fmt.Println(string(data))

}

func ArticleSearch(query string) []*Article {
	fmt.Printf("ArticleSearch: %s\n", query)
	var err error
	client, err := elastic.NewSimpleClient(elastic.SetURL("http://127.0.0.1:9200/"))
	if err != nil {
		// Handle error
		fmt.Println(err.Error())
		return nil
	}
	defer client.Stop()

	se_index := "blogweb_article"
	se_type := "doc"

	//条件查询
	formatS := `{
  "multi_match": {
      "query":    "%s",
      "fields":   [ "short", "content"]
  }
}`
	rawQ := elastic.NewRawStringQuery(
		fmt.Sprintf(formatS, query))
	DumpQuery(rawQ.Source())
	searchResult, err := client.Search(se_index).Type(se_type).
		Query(rawQ).
		From(0).Size(30).
		Do(context.Background())
	if err != nil {
		// Handle error
		fmt.Println(err.Error())
		return nil
	}
	fmt.Printf("Query took %d milliseconds. total: %d\n", searchResult.TookInMillis, searchResult.Hits.TotalHits)

	result := []*Article{}
	var ttyp ArticleSE
	for _, item := range searchResult.Each(reflect.TypeOf(ttyp)) {
		if t, ok := item.(ArticleSE); ok {
			result = append(result, ArticleFromSearchEntity(&t))
		}
	}
	return result
}

测试查询代码

package main

import (
	"fmt"
	"grpcdemo/models"
)

func main() {
	for _, item := range models.ArticleSearch("golang") {
		fmt.Printf("%d, %s, %s\n", item.Id, item.Short, item.Content)
	}
	for _, item := range models.ArticleSearch("理解") {
		fmt.Printf("%d, %s, %s\n", item.Id, item.Short, item.Content)
	}
	for _, item := range models.ArticleSearch("解") {
		fmt.Printf("%d, %s, %s\n", item.Id, item.Short, item.Content)
	}
}

运行结果

PS D:\mygo\src\grpcdemo\esquery> .\esquery.exe
ArticleSearch: golang
{
  "multi_match": {
    "fields": [
      "short",
      "content"
    ],
    "query": "golang"
  }
}
Query took 1 milliseconds. total: 1
3, 学习golang, etcd学习
ArticleSearch: 理解
{
  "multi_match": {
    "fields": [
      "short",
      "content"
    ],
    "query": "理解"
  }
}
Query took 1 milliseconds. total: 1
2, world, 理解Elastic Search
ArticleSearch: 解
{
  "multi_match": {
    "fields": [
      "short",
      "content"
    ],
    "query": "解"
  }
}
Query took 1 milliseconds. total: 0

6.生产环境如何获取搜索结果

  • 从ES获取的数据，有可能不是最新
  • ES展现列表页，详情页读数据库
  • 从ES获得id，从数据库获得数据
推荐阅读