ES全文检索
ES安装
-
拷贝elasticsearch-5.6.4.rpm到/opt目录下
-
配置jdk
-
vim /etc/sysconfig/elasticsearch 中修改JAVA_HOME路径的路径
-
9 JAVA_HOME=/opt/jdk1.8.0_152
-
核心文件
-
vim /etc/elasticsearch/elasticsearch.yml
数据文件路径
/var/lib/elasticsearch/
日志文件路径
/var/log/elasticsearch/elasticsearch.log
修改配置文件
-
vim /etc/elasticsearch/elasticsearch.yml
-
修改yml配置的注意事项:
每行必须顶格,不能有空格
“:”后面必须有一个空格
集群名称,同一集群名称必须相同
-
-
单个节点名称
-
-
网络部分 改为当前的ip地址 ,端口号保持默认9200就行
-
-
把bootstrap自检程序关掉
bootstrap.system_call_filter: false
-
-
自发现配置:新节点向集群报到的主机名,也可以写ip地址
-
修改linux配置
-
为什么要修改linux配置?
默认elasticsearch是单机访问模式,就是只能自己访问自己。
但是我们之后一定会设置成允许应用服务器通过网络方式访问。这时,elasticsearch就会因为嫌弃单机版的低端默认配置而报错,甚至无法启动。
所以我们在这里就要把服务器的一些限制打开,能支持更多并发。
-
问题1:max file descriptors [4096] for elasticsearch process likely too low, increase to at least [65536] elasticsearch
-
原因:系统允许 Elasticsearch 打开的最大文件数需要修改成65536
解决:vi /etc/security/limits.conf
添加内容:
* soft nofile 65536
* hard nofile 131072
* soft nproc 2048
* hard nproc 65536
注意:“*” 不要省略掉
重启linux
-
vim /etc/elasticsearch/jvm.options
调整一下elasticsearch 的配置,将启动的内存调整为512m
# reboot
安装kibana
-
拷贝kibana-5.6.4-linux-x86_64.tar 到/opt下
解压缩
进入kibana主目录的config目录下
-
/opt/kibana-5.6.4-linux-x86_64/config
-
vim kibana.yml
-
-
启动
在 kibana主目录bin目录下执行
nohup ./kibana &
然后ctrl+c退出
执行ps -ef
-
在Java中的运用
(1)导入es相关依赖
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-elasticsearch</artifactId>
</dependency>
<!-- https://mvnrepository.com/artifact/io.searchbox/jest -->
<dependency>
<groupId>io.searchbox</groupId>
<artifactId>jest</artifactId>
</dependency>
<!-- https://mvnrepository.com/artifact/net.java.dev.jna/jna -->
<dependency>
<groupId>net.java.dev.jna</groupId>
<artifactId>jna</artifactId>
</dependency>
(2)在测试类中测试ES
-
application.properties
-
server.port=8085 logging.level.root=error spring.dubbo.application.name=list-service spring.dubbo.registry.protocol=zookeeper spring.dubbo.registry.address=192.168.113.132:2181 spring.dubbo.base-package=com.atguan.gmall spring.dubbo.protocol.name=dubbo spring.datasource.url=jdbc:mysql://192.168.113.132:3306/gmall?characterEncoding=UTF-8 spring.datasource.username=root spring.datasource.password=123456 spring.datasource.driver-class-name=com.mysql.jdbc.Driver #mybatis mybatis.mapper-locations=classpath:mapper/*Mapper.xml mybatis.configuration.mapUnderscoreToCamelCase=true # redis spring.redis.host=192.168.113.132 spring.redis.port=6379 spring.redis.database=0 spring.elasticsearch.jest.uris=http://192.168.113.132:9200
-
在测试类中添加
-
/** * 测试能否与es连通 */ @Test public void testES() throws IOException { //GET /movie_chn/movie/_search String query = "{\n" + " \"query\": {\n" + " \"term\": {\n" + " \"actorList.name\": \"张译\"\n" + " }\n" + " }\n" + "}"; //查询get Search search = new Search.Builder(query).addIndex("movie_chn").addType("movie").build(); //执行 SearchResult searchResult = jestClient.execute(search); //获取执行结果 List<SearchResult.Hit<Map, Void>> hits = searchResult.getHits(Map.class); for (SearchResult.Hit<Map, Void> hit : hits) { Map source = hit.source; System.err.println(source.get("name")); } }
(3)将数据加入es中
在es中自定义Mapping
-
PUT gmall { "mappings": { "SkuInfo": { "properties": { "id": { "type": "keyword" ,"index": false }, "price": { "type": "double" }, "skuName": { "type": "text" ,"analyzer": "ik_max_word" }, "catalog3Id": { "type": "keyword" }, "skuDefaultImg": { "type": "keyword" ,"index": false }, "skuAttrValueList": { "properties": { "valueId":{ "type":"keyword" } } } } } } }
保存数据到gmall/SkuInfo中
-
定义实体类
-
@Data public class SkuLsInfo implements Serializable { String id; BigDecimal price; String skuName; String catalog3Id; String skuDefaultImg; Long hotScore=0L; List<SkuLsAttrValue> skuAttrValueList; }
-
@Data public class SkuLsAttrValue implements Serializable { String valueId; }
-
-
service.impl
-
@Service public class ListServiceImpl implements ListService { @Autowired private JestClient jestClient; public static final String ES_INDEX="gmall"; public static final String ES_TYPE="SkuInfo"; @Override public void saveSkuLsInfo(SkuLsInfo skuLsInfo) { Index index = new Index.Builder(skuLsInfo).index(ES_INDEX).type(ES_TYPE).id(skuLsInfo.getId()).build(); try { jestClient.execute(index); } catch (IOException e) { e.printStackTrace(); } } }
-
-
Controller
-
商品上架 @RequestMapping("/onSale") public void onSale(@RequestParam String skuId) { SkuLsInfo skuLsInfo = new SkuLsInfo(); //给skuLsInfo赋值 SkuInfo skuInfo = manageService.getSkuInfo(skuId); //属性对拷 BeanUtils.copyProperties(skuInfo,skuLsInfo); listService.saveSkuLsInfo(skuLsInfo); }
-
-
编写dsl语句
-
GET gmall/SkuInfo/_search { "query": { "bool": { "filter": [{"term":{"catalog3Id":"61"}}, {"term":{"skuAttrValueList.valueId":"82"}} ], "must": [ { "match": { "skuName": "华为" } } ] } }, "highlight": { "pre_tags": ["<span style=color:red>"], "post_tags": ["</span>"], "fields": {"skuName": {}} }, "from": 0 ,"size": 20 ,"sort": [ { "hotScore": { "order": "desc" } } ], "aggs": { "groupby_attr": { "terms": { "field": "skuAttrValueList.valueId" } } } }
-
(4)动态生成dsl语句
-
面向对象:将用户查询条件封装为对象
-
@Data public class SkuLsParams implements Serializable { String keyword; String catalog3Id; String[] valueId; int pageNo=1; int pageSize=20; }
-
@Data public class SkuLsResult implements Serializable { List<SkuLsInfo> skuLsInfoList; long total; long totalPages; List<String> attrValueIdList; }
-
-
Service添加方法
-
/** * 全文检索 * @param skuLsParams * @return */ SkuLsResult search(SkuLsParams skuLsParams);
-
-
service.impl
-
@Override public SkuLsResult search(SkuLsParams skuLsParams) { //定义dsl语句 String query = makeQueryStringForSearch(skuLsParams); //查询 Search search = new Search.Builder(query).addIndex(ES_INDEX).addType(ES_TYPE).build(); SearchResult result = null; //执行 try { result = jestClient.execute(search); System.err.println(result); } catch (IOException e) { e.printStackTrace(); } SkuLsResult skuLsResult = makeResultForSearch(result,skuLsParams); return skuLsResult; } private SkuLsResult makeResultForSearch(SearchResult result, SkuLsParams skuLsParams) { SkuLsResult skuLsResult = new SkuLsResult(); List<SkuLsInfo> list = new ArrayList<>(); //给集合赋值 List<SearchResult.Hit<SkuLsInfo, Void>> hits = result.getHits(SkuLsInfo.class); for (SearchResult.Hit<SkuLsInfo, Void> hit : hits) { SkuLsInfo skuLsInfo = hit.source; if (hit.highlight != null && hit.highlight.size() > 0) { Map<String, List<String>> highlight = hit.highlight; List<String> skuName = highlight.get("skuName"); //高亮sku String s = skuName.get(0); skuLsInfo.setSkuName(s); } list.add(skuLsInfo); } skuLsResult.setSkuLsInfoList(list); //total skuLsResult.setTotal(skuLsResult.getTotal()); //page long pages = (result.getTotal()+skuLsParams.getPageSize()-1)/skuLsParams.getPageSize(); skuLsResult.setTotalPages(pages); //平台属性 MetricAggregation aggregations = result.getAggregations(); TermsAggregation groupby_attr = aggregations.getTermsAggregation("groupby_attr"); List<TermsAggregation.Entry> buckets = groupby_attr.getBuckets(); List<String> attrValueIdList = new ArrayList<>(); for (TermsAggregation.Entry bucket : buckets) { String valueId = bucket.getKey(); if (valueId != null) { attrValueIdList.add(valueId); } } skuLsResult.setAttrValueIdList(attrValueIdList); return skuLsResult; } private String makeQueryStringForSearch(SkuLsParams skuLsParams) { /**dsl语句: * GET gmall/SkuInfo/_search * { * "query": { * "bool": { * "filter": [{"term":{"catalog3Id":"61"}}, * {"term":{"skuAttrValueList.valueId":"82"}} * ], * "must": [ * { * "match": { * "skuName": "华为" * } * } * ] * } * }, * "highlight": { * "pre_tags": ["<span style=color:red>"], * "post_tags": ["</span>"], * "fields": {"skuName": {}} * }, * "from": 0 * ,"size": 20 * ,"sort": [ * { * "hotScore": { * "order": "desc" * } * } * ], * "aggs": { * "groupby_attr": { * "terms": { * "field": "skuAttrValueList.valueId" * } * } * } * } */ //定义一个查询器 SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); //创建bool BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); //判断三级分类id if (skuLsParams.getCatalog3Id() != null && skuLsParams.getCatalog3Id().length() > 0) { //创建term TermQueryBuilder catalog3Id = new TermQueryBuilder("catalog3Id", skuLsParams.getCatalog3Id()); //创建filter boolQueryBuilder.filter(catalog3Id); } //判断平台属性值valueId if (skuLsParams.getValueId() != null && skuLsParams.getValueId().length > 0) { for (String valueId : skuLsParams.getValueId()) { //创建term TermQueryBuilder termQueryBuilder= new TermQueryBuilder("skuAttrValueList.valueId", valueId); //创建filter boolQueryBuilder.filter(termQueryBuilder); } } //判断keyword是否为空 if (skuLsParams.getKeyword() != null && skuLsParams.getKeyword().length() > 0) { MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder("skuName",skuLsParams.getKeyword()); //创建must boolQueryBuilder.must(matchQueryBuilder); //设置高亮 HighlightBuilder highlighter = searchSourceBuilder.highlighter(); //设置高亮规则 highlighter.field("skuName"); highlighter.preTags("<span style=color:red>"); highlighter.postTags("</span>"); //放入查询器 searchSourceBuilder.highlight(highlighter); } //query searchSourceBuilder.query(boolQueryBuilder); //设置分页 int from = (skuLsParams.getPageNo()-1)*skuLsParams.getPageSize(); searchSourceBuilder.from(from); searchSourceBuilder.size(skuLsParams.getPageSize()); //设置排序 searchSourceBuilder.sort("hotScore", SortOrder.DESC); //设置聚合 TermsBuilder groupby_attr = AggregationBuilders.terms("groupby_attr"); groupby_attr.field("skuAttrValueList.valueId"); searchSourceBuilder.aggregation(groupby_attr); String query = searchSourceBuilder.toString(); System.err.println(query); return query; }
-
评论区