fanxb/bookmark
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 6 Wiki Activity

feat:增加拼音检索功能

Browse Source
This commit is contained in:
fanxb 2020-03-29 17:08:18 +08:00
parent afb9886756
commit 09795f2d59
18 changed files with 552 additions and 394 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
bookMarkService/business/bookmark/src/main/java/com/fanxb/bookmark/business/bookmark/consumer/BookmarkDeleteEsConsumer.java Normal file
View File

@ -0,0 +1,33 @@
package com.fanxb.bookmark.business.bookmark.consumer;
import cn.hutool.core.collection.CollectionUtil;
import com.alibaba.fastjson.JSONArray;
import com.fanxb.bookmark.common.annotation.MqConsumer;
import com.fanxb.bookmark.common.constant.EsConstant;
import com.fanxb.bookmark.common.constant.RedisConstant;
import com.fanxb.bookmark.common.entity.redis.RedisConsumer;
import com.fanxb.bookmark.common.util.EsUtil;
import org.springframework.beans.factory.annotation.Autowired;
import java.util.List;
/**
* Created with IntelliJ IDEA
* Created By Fxb
* Date: 2020/3/29
* Time: 13:08
*/
@MqConsumer(RedisConstant.BOOKMARK_DELETE_ES)
public class BookmarkDeleteEsConsumer implements RedisConsumer {
@Autowired
private EsUtil esUtil;
@Override
public void deal(String message) {
List<String> strings = JSONArray.parseArray(message, String.class);
if (CollectionUtil.isEmpty(strings)) {
return;
}
esUtil.deleteBatch(EsConstant.BOOKMARK_INDEX, strings);
}
}

42
bookMarkService/business/bookmark/src/main/java/com/fanxb/bookmark/business/bookmark/consumer/BookmarkInsertEsConsumer.java Normal file
View File

@ -0,0 +1,42 @@
package com.fanxb.bookmark.business.bookmark.consumer;
import cn.hutool.core.collection.CollectionUtil;
import com.alibaba.fastjson.JSONArray;
import com.fanxb.bookmark.business.bookmark.entity.BookmarkEs;
import com.fanxb.bookmark.common.annotation.MqConsumer;
import com.fanxb.bookmark.common.constant.EsConstant;
import com.fanxb.bookmark.common.constant.RedisConstant;
import com.fanxb.bookmark.common.entity.Bookmark;
import com.fanxb.bookmark.common.entity.EsEntity;
import com.fanxb.bookmark.common.entity.redis.RedisConsumer;
import com.fanxb.bookmark.common.util.EsUtil;
import org.springframework.beans.factory.annotation.Autowired;
import java.util.List;
import java.util.stream.Collectors;
/**
* 插入/更新数据到es中
* Created with IntelliJ IDEA
* Created By Fxb
* Date: 2020/3/29
* Time: 11:34
*/
@MqConsumer(RedisConstant.BOOKMARK_INSERT_ES)
public class BookmarkInsertEsConsumer implements RedisConsumer {
@Autowired
private EsUtil esUtil;
@Override
public void deal(String message) {
List<Bookmark> bookmarks = JSONArray.parseArray(message, Bookmark.class);
if (CollectionUtil.isEmpty(bookmarks)) {
return;
}
List<EsEntity<BookmarkEs>> esList = bookmarks.stream()
.map(item -> new EsEntity<>(item.getBookmarkId().toString(), new BookmarkEs(item))).collect(Collectors.toList());
esUtil.insertBatch(EsConstant.BOOKMARK_INDEX, esList);
}
}

45
bookMarkService/business/bookmark/src/main/java/com/fanxb/bookmark/business/bookmark/consumer/PinyinUpdateConsumer.java Normal file
View File

@ -0,0 +1,45 @@
package com.fanxb.bookmark.business.bookmark.consumer;
import cn.hutool.core.collection.CollectionUtil;
import com.alibaba.fastjson.JSONArray;
import com.fanxb.bookmark.business.bookmark.dao.BookmarkDao;
import com.fanxb.bookmark.business.bookmark.service.PinYinService;
import com.fanxb.bookmark.common.annotation.MqConsumer;
import com.fanxb.bookmark.common.constant.RedisConstant;
import com.fanxb.bookmark.common.entity.Bookmark;
import com.fanxb.bookmark.common.entity.redis.RedisConsumer;
import com.fanxb.bookmark.common.entity.redis.UserBookmarkUpdate;
import com.fanxb.bookmark.common.util.RedisUtil;
import org.springframework.beans.factory.annotation.Autowired;
import java.util.List;
import java.util.stream.Collectors;
/**
* 平应转换
* Created By Fxb
* Date: 2020/3/29
* Time: 13:01
*/
@MqConsumer(RedisConstant.BOOKMARK_PINYIN_CHANGE)
public class PinyinUpdateConsumer implements RedisConsumer {
@Autowired
private PinYinService pinYinService;
@Autowired
private BookmarkDao bookmarkDao;
@Override
public void deal(String message) {
List<Bookmark> bookmarks = JSONArray.parseArray(message, Bookmark.class);
if (CollectionUtil.isEmpty(bookmarks)) {
return;
}
List<String> resList = pinYinService.changeStrings(bookmarks.stream().map(Bookmark::getName).collect(Collectors.toList()));
for (int i = 0, size = bookmarks.size(); i < size; i++) {
bookmarkDao.updateSearchKey(bookmarks.get(i).getBookmarkId(), resList.get(i));
}
//更新本用户书签更新时间
RedisUtil.addToMq(RedisConstant.BOOKMARK_UPDATE_TIME, new UserBookmarkUpdate(bookmarks.get(0).getUserId()));
}
}

3
bookMarkService/business/bookmark/src/main/java/com/fanxb/bookmark/business/bookmark/controller/BookmarkBackupController.java
View File

@ -1,6 +1,7 @@
package com.fanxb.bookmark.business.bookmark.controller;
import com.fanxb.bookmark.business.bookmark.service.BookmarkBackupService;
import com.fanxb.bookmark.business.bookmark.service.impl.BookmarkBackupServiceImpl;
import com.fanxb.bookmark.common.entity.Result;
import com.fanxb.bookmark.common.util.ThreadPoolUtil;
import org.springframework.beans.factory.annotation.Autowired;
@ -23,7 +24,7 @@ public class BookmarkBackupController {
private BookmarkBackupService backupService;
@Autowired
public BookmarkBackupController(BookmarkBackupService backupService) {
public BookmarkBackupController(BookmarkBackupServiceImpl backupService) {
this.backupService = backupService;
}

7
bookMarkService/business/bookmark/src/main/java/com/fanxb/bookmark/business/bookmark/controller/BookmarkController.java
View File

@ -3,11 +3,11 @@ package com.fanxb.bookmark.business.bookmark.controller;
import com.fanxb.bookmark.business.bookmark.entity.BatchDeleteBody;
import com.fanxb.bookmark.business.bookmark.entity.BookmarkEs;
import com.fanxb.bookmark.business.bookmark.entity.MoveNodeBody;
import com.fanxb.bookmark.business.bookmark.service.BookmarkBackupService;
import com.fanxb.bookmark.business.bookmark.service.BookmarkService;
import com.fanxb.bookmark.business.bookmark.service.PinYinService;
import com.fanxb.bookmark.common.entity.Bookmark;
import com.fanxb.bookmark.common.entity.Result;
import com.fanxb.bookmark.common.entity.UserContext;
import com.fanxb.bookmark.common.util.UserContextHolder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
@ -25,7 +25,8 @@ import java.util.List;
@RestController
@RequestMapping("/bookmark")
public class BookmarkController {
@Autowired
private BookmarkBackupService bookmarkBackupService;
@Autowired
private BookmarkService bookmarkService;
@Autowired
@ -143,7 +144,7 @@ public class BookmarkController {
*/
@PostMapping("/syncBookmark")
public Result syncBookmark() {
bookmarkService.syncUserBookmark(UserContextHolder.get().getUserId());
bookmarkBackupService.syncUserBookmark(UserContextHolder.get().getUserId());
return Result.success(null);
}

27
bookMarkService/business/bookmark/src/main/java/com/fanxb/bookmark/business/bookmark/dao/BookmarkBackupDao.java
View File

@ -1,27 +0,0 @@
package com.fanxb.bookmark.business.bookmark.dao;
import com.fanxb.bookmark.common.entity.Bookmark;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Select;
import java.util.List;
/**
* Created with IntelliJ IDEA
* Created By Fxb
* Date: 2019/11/12
* Time: 0:24
*/
@Mapper
public interface BookmarkBackupDao {
/**
* 分页获取所有的书签
* @param size 大小
* @param startIndex 开始下标
* @return
*/
@Select("select * from bookmark order by bookmarkId limit ${startIndex},${size}")
List<Bookmark> getBookmarkListPage(@Param("size") int size,@Param("startIndex") int startIndex);
}

10
bookMarkService/business/bookmark/src/main/java/com/fanxb/bookmark/business/bookmark/dao/BookmarkDao.java
View File

@ -184,4 +184,14 @@ public interface BookmarkDao extends BaseMapper<Bookmark> {
@Update("update bookmark set searchKey=#{searchKey} where bookmarkId=#{bookmarkId}")
void updateSearchKey(@Param("bookmarkId") int bookmarkId, @Param("searchKey") String searchKey);
/**
* 分页获取所有的书签
*
* @param size 大小
* @param startIndex 开始下标
* @return
*/
@Select("select * from bookmark order by bookmarkId limit ${startIndex},${size}")
List<Bookmark> getBookmarkListPage(@Param("size") int size, @Param("startIndex") int startIndex);
}

51
bookMarkService/business/bookmark/src/main/java/com/fanxb/bookmark/business/bookmark/service/BookmarkBackupService.java
View File

@ -1,39 +1,13 @@
package com.fanxb.bookmark.business.bookmark.service;
import com.fanxb.bookmark.business.bookmark.dao.BookmarkBackupDao;
import com.fanxb.bookmark.business.bookmark.dao.BookmarkDao;
import com.fanxb.bookmark.business.bookmark.entity.BookmarkEs;
import com.fanxb.bookmark.common.constant.EsConstant;
import com.fanxb.bookmark.common.entity.Bookmark;
import com.fanxb.bookmark.common.entity.EsEntity;
import com.fanxb.bookmark.common.util.EsUtil;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
/**
* Created with IntelliJ IDEA
* Created By Fxb
* Date: 2019/11/12
* Time: 0:22
*
* @author fanxb
* Date: 2020/3/29
* Time: 12:43
*/
@Service
public class BookmarkBackupService {
public interface BookmarkBackupService {
@Autowired
private BookmarkBackupDao bookmarkBackupDao;
@Autowired
private EsUtil esUtil;
/**
* 一次同步BACKUP_SIZE条到es中
*/
private static final int BACKUP_SIZE = 500;
/**
* 功能描述: 将mysql数据同步到es中
@ -41,14 +15,13 @@ public class BookmarkBackupService {
* @author fanxb
* @date 2019/11/12 0:22
*/
public void backupToEs() {
int start = 0;
List<Bookmark> list;
while ((list = bookmarkBackupDao.getBookmarkListPage(BACKUP_SIZE, start)).size() != 0) {
List<EsEntity<BookmarkEs>> batchList = new ArrayList<>(list.size());
list.forEach(item -> batchList.add(new EsEntity<>(item.getBookmarkId().toString(), new BookmarkEs(item))));
esUtil.insertBatch(EsConstant.BOOKMARK_INDEX, batchList);
start += BACKUP_SIZE;
}
}
void backupToEs();
/**
* Description: 将某个用户的书签数据mysql同步到es中
*
* @author fanxb
* @date 2019/7/26 11:27
*/
void syncUserBookmark(int userId);
}

345
bookMarkService/business/bookmark/src/main/java/com/fanxb/bookmark/business/bookmark/service/BookmarkService.java
View File

@ -1,184 +1,25 @@
package com.fanxb.bookmark.business.bookmark.service;
import com.fanxb.bookmark.business.bookmark.dao.BookmarkDao;
import com.fanxb.bookmark.business.bookmark.entity.BookmarkEs;
import com.fanxb.bookmark.business.bookmark.entity.MoveNodeBody;
import com.fanxb.bookmark.common.constant.EsConstant;
import com.fanxb.bookmark.common.constant.RedisConstant;
import com.fanxb.bookmark.common.entity.Bookmark;
import com.fanxb.bookmark.common.entity.EsEntity;
import com.fanxb.bookmark.common.entity.redis.UserBookmarkUpdate;
import com.fanxb.bookmark.common.exception.FormDataException;
import com.fanxb.bookmark.common.util.EsUtil;
import com.fanxb.bookmark.common.util.RedisUtil;
import com.fanxb.bookmark.common.util.ThreadPoolUtil;
import com.fanxb.bookmark.common.util.UserContextHolder;
import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.io.InputStream;
import java.util.*;
import java.util.stream.Collectors;
import java.util.List;
import java.util.Map;
/**
* 类功能简述：
* 类功能详述：
*
* @author fanxb
* @date 2019/7/8 15:00
* Created with IntelliJ IDEA
* Created By Fxb
* Date: 2020/3/29
* Time: 12:25
*/
@Service
@Slf4j
public class BookmarkService {
public interface BookmarkService {
/**
* chrome导出书签tag
*/
private static final String DT = "dt";
private static final String A = "a";
@Autowired
private BookmarkDao bookmarkDao;
@Autowired
private StringRedisTemplate redisTemplate;
@Autowired
private PinYinService pinYinService;
@Autowired
private EsUtil esUtil;
/**
* Description: 解析书签文件
*
* @param stream 输入流
* @param path 存放路径
* @author fanxb
* @date 2019/7/9 18:44
*/
@Transactional(rollbackFor = Exception.class)
public void parseBookmarkFile(int userId, InputStream stream, String path) throws Exception {
Document doc = Jsoup.parse(stream, "utf-8", "");
Elements elements = doc.select("html>body>dl>dt");
//获取当前层sort最大值
Integer sortBase = bookmarkDao.selectMaxSort(userId, path);
if (sortBase == null) {
sortBase = 0;
}
int count = 0;
// 將要插入es的书签数据放到list中,最后一次插入，尽量避免mysql回滚了，但是es插入了
List<EsEntity<BookmarkEs>> insertEsList = new ArrayList<>();
for (int i = 0, length = elements.size(); i < length; i++) {
if (i == 0) {
Elements firstChildren = elements.get(0).child(1).children();
count = firstChildren.size();
for (int j = 0; j < count; j++) {
dealBookmark(userId, firstChildren.get(j), path, sortBase + j, insertEsList);
}
} else {
dealBookmark(userId, elements.get(i), path, sortBase + count + i - 1, insertEsList);
}
}
redisTemplate.opsForList().leftPush(RedisConstant.BOOKMARK_UPDATE_TIME, new UserBookmarkUpdate(userId, System.currentTimeMillis()).toString());
esUtil.insertBatch(EsConstant.BOOKMARK_INDEX, insertEsList);
//异步更新searchKey
ThreadPoolUtil.execute((() -> {
List<BookmarkEs> list = insertEsList.stream().map(EsEntity::getData).collect(Collectors.toList());
List<String> resList = pinYinService.changeStrings(list.stream().map(BookmarkEs::getName).collect(Collectors.toList()));
for (int i = 0, size = list.size(); i < size; i++) {
bookmarkDao.updateSearchKey(list.get(i).getBookmarkId(), resList.get(i));
}
}));
}
/**
* Description: 处理html节点，解析出文件夹和书签
*
* @param ele 待处理节点
* @param path 节点路径，不包含自身
* @param sort 当前层级中的排序序号
* @author fanxb
* @date 2019/7/8 14:49
*/
private void dealBookmark(int userId, Element ele, String path, int sort, List<EsEntity<BookmarkEs>> insertList) {
if (!DT.equalsIgnoreCase(ele.tagName())) {
return;
}
Element first = ele.child(0);
if (A.equalsIgnoreCase(first.tagName())) {
//说明为链接
Bookmark node = new Bookmark(userId, path, first.ownText(), first.attr("href"), first.attr("icon")
, Long.parseLong(first.attr("add_date")) * 1000, sort);
//存入数据库
insertOne(node);
insertList.add(new EsEntity<>(node.getBookmarkId().toString(), new BookmarkEs(node)));
} else {
//说明为文件夹
Bookmark node = new Bookmark(userId, path, first.ownText(), Long.parseLong(first.attr("add_date")) * 1000, sort);
Integer sortBase = 0;
//同名文件夹将会合并
if (insertOne(node)) {
sortBase = bookmarkDao.selectMaxSort(node.getUserId(), path);
if (sortBase == null) {
sortBase = 0;
}
}
String childPath = path + "." + node.getBookmarkId();
Elements children = ele.child(1).children();
for (int i = 0, size = children.size(); i < size; i++) {
dealBookmark(userId, children.get(i), childPath, sortBase + i + 1, insertList);
}
}
}
/**
* Description: 插入一条书签，如果已经存在同名书签将跳过
*
* @param node node
* @return boolean 如果已经存在返回true，否则false
* @author fanxb
* @date 2019/7/8 17:25
*/
private boolean insertOne(Bookmark node) {
//先根据name,userId,parentId获取此节点id
Integer id = bookmarkDao.selectIdByUserIdAndNameAndPath(node.getUserId(), node.getName(), node.getPath());
if (id == null) {
bookmarkDao.insertOne(node);
return false;
} else {
node.setBookmarkId(id);
return true;
}
}
/**
* 功能描述: 获取某个用户的书签map
*
* @param userId userId
* @return java.util.Map<java.lang.String, java.util.List < com.fanxb.bookmark.common.entity.Bookmark>>
* @author fanxb
* @date 2019/12/14 0:02
*/
public Map<String, List<Bookmark>> getOneBookmarkTree(int userId) {
List<Bookmark> list = bookmarkDao.getListByUserId(userId);
Map<String, List<Bookmark>> map = new HashMap<>(50);
list.forEach(item -> {
map.computeIfAbsent(item.getPath(), k -> new ArrayList<>());
map.get(item.getPath()).add(item);
});
return map;
}
static final String DT = "dt";
static final String A = "a";
/**
* Description: 根据userId和path获取书签列表
@ -189,10 +30,47 @@ public class BookmarkService {
* @author fanxb
* @date 2019/7/15 13:40
*/
public List<Bookmark> getBookmarkListByPath(int userId, String path) {
return bookmarkDao.getListByUserIdAndPath(userId, path);
}
List<Bookmark> getBookmarkListByPath(int userId, String path);
/**
* 功能描述: 获取某个用户的书签map
*
* @param userId userId
* @return java.util.Map<java.lang.String, java.util.List < com.fanxb.bookmark.common.entity.Bookmark>>
* @author fanxb
* @date 2019/12/14 0:02
*/
Map<String, List<Bookmark>> getOneBookmarkTree(int userId);
/**
* Description: 解析书签文件
*
* @param stream 输入流
* @param path 存放路径
* @author fanxb
* @date 2019/7/9 18:44
*/
void parseBookmarkFile(int userId, InputStream stream, String path) throws Exception;
/**
* Description: 详情
*
* @param bookmark 插入一条记录
* @return com.fanxb.bookmark.common.entity.Bookmark
* @author fanxb
* @date 2019/7/12 17:18
*/
Bookmark addOne(Bookmark bookmark);
/**
* Description: 编辑某个用户的某个书签
*
* @param userId userId
* @param bookmark bookmark
* @author fanxb
* @date 2019/7/17 14:42
*/
void updateOne(int userId, Bookmark bookmark);
/**
* Description: 批量删除书签
@ -203,94 +81,17 @@ public class BookmarkService {
* @author fanxb
* @date 2019/7/12 14:09
*/
@Transactional(rollbackFor = Exception.class)
public void batchDelete(int userId, List<Integer> folderIdList, List<Integer> bookmarkIdList) {
Set<Integer> set = new HashSet<>();
for (Integer item : folderIdList) {
set.addAll(bookmarkDao.getChildrenBookmarkId(userId, item));
bookmarkDao.deleteUserFolder(userId, item);
bookmarkIdList.add(item);
}
if (bookmarkIdList.size() > 0) {
bookmarkDao.deleteUserBookmark(userId, bookmarkIdList);
}
set.addAll(bookmarkIdList);
redisTemplate.opsForList().leftPush(RedisConstant.BOOKMARK_UPDATE_TIME, new UserBookmarkUpdate(userId, System.currentTimeMillis()).toString());
//es 中批量删除
esUtil.deleteBatch(EsConstant.BOOKMARK_INDEX, set);
}
void batchDelete(int userId, List<Integer> folderIdList, List<Integer> bookmarkIdList);
/**
* Description: 详情
*
* @param bookmark 插入一条记录
* @return com.fanxb.bookmark.common.entity.Bookmark
* @author fanxb
* @date 2019/7/12 17:18
*/
@Transactional(rollbackFor = Exception.class)
public Bookmark addOne(Bookmark bookmark) {
int userId = UserContextHolder.get().getUserId();
Integer sort = bookmarkDao.selectMaxSort(userId, bookmark.getPath());
bookmark.setSort(sort == null ? 1 : sort + 1);
bookmark.setUserId(userId);
bookmark.setCreateTime(System.currentTimeMillis());
bookmark.setAddTime(bookmark.getCreateTime());
bookmark.setSearchKey(pinYinService.changeString(bookmark.getName()));
try {
bookmarkDao.insertOne(bookmark);
} catch (DuplicateKeyException e) {
throw new FormDataException("同级目录下不能存在相同名称的数据");
}
//如果是书签，插入到es中
if (bookmark.getType() == 0) {
esUtil.insertOrUpdateOne(EsConstant.BOOKMARK_INDEX,
new EsEntity<>(bookmark.getBookmarkId().toString(), new BookmarkEs(bookmark)));
}
redisTemplate.opsForList().leftPush(RedisConstant.BOOKMARK_UPDATE_TIME, new UserBookmarkUpdate(userId, System.currentTimeMillis()).toString());
return bookmark;
}
/**
* Description: 编辑某个用户的某个书签
* 功能描述: 移动一个节点
*
* @param userId userId
* @param bookmark bookmark
* @author fanxb
* @date 2019/7/17 14:42
* @param body body
* @author 123
* @date 2020/3/29 12:30
*/
@Transactional(rollbackFor = Exception.class)
public void updateOne(int userId, Bookmark bookmark) {
bookmark.setUserId(userId);
bookmark.setSearchKey(pinYinService.changeString(bookmark.getName()));
bookmarkDao.editBookmark(bookmark);
if (bookmark.getType() == 0) {
esUtil.insertOrUpdateOne(EsConstant.BOOKMARK_INDEX,
new EsEntity<>(bookmark.getBookmarkId().toString(), new BookmarkEs(bookmark)));
}
redisTemplate.opsForList().leftPush(RedisConstant.BOOKMARK_UPDATE_TIME, new UserBookmarkUpdate(userId, System.currentTimeMillis()).toString());
}
@Transactional(rollbackFor = Exception.class)
public void moveNode(int userId, MoveNodeBody body) {
if (body.getSort() == -1) {
Integer max = bookmarkDao.selectMaxSort(userId, body.getTargetPath());
body.setSort(max == null ? 1 : max);
} else {
//更新目标节点的sort
bookmarkDao.sortPlus(userId, body.getTargetPath(), body.getSort());
}
//如果目标位置和当前位置不在一个层级中需要更新子节点的path
if (!body.getTargetPath().equals(body.getSourcePath())) {
bookmarkDao.updateChildrenPath(userId, body.getSourcePath() + "." + body.getBookmarkId()
, body.getTargetPath() + "." + body.getBookmarkId());
}
//更新被移动节点的path和sort
bookmarkDao.updatePathAndSort(userId, body.getBookmarkId(), body.getTargetPath(), body.getSort());
redisTemplate.opsForList().leftPush(RedisConstant.BOOKMARK_UPDATE_TIME, new UserBookmarkUpdate(userId, System.currentTimeMillis()).toString());
log.info("{},从{}移动到{},sort:{}", userId, body.getSourcePath(), body.getTargetPath(), body.getSort());
}
void moveNode(int userId, MoveNodeBody body);
/**
* Description: 根据context搜索
@ -300,39 +101,5 @@ public class BookmarkService {
* @author fanxb
* @date 2019/7/25 10:45
*/
public List<BookmarkEs> searchUserBookmark(int userId, String context) {
BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
boolQueryBuilder.must(QueryBuilders.termQuery("userId", userId));
boolQueryBuilder.must(QueryBuilders.multiMatchQuery(context, "name", "url"));
SearchSourceBuilder builder = new SearchSourceBuilder();
builder.size(5);
builder.query(boolQueryBuilder);
return esUtil.search(EsConstant.BOOKMARK_INDEX, builder, BookmarkEs.class);
}
/**
* Description: 将某个用户的书签数据mysql同步到es中
*
* @author fanxb
* @date 2019/7/26 11:27
*/
public void syncUserBookmark(int userId) {
//删除旧的数据
esUtil.deleteByQuery(EsConstant.BOOKMARK_INDEX, new TermQueryBuilder("userId", userId));
int index = 0;
int size = 500;
List<EsEntity<BookmarkEs>> res = new ArrayList<>();
do {
res.clear();
bookmarkDao.selectBookmarkEsByUserIdAndType(userId, 0, index, size)
.forEach(item -> res.add(new EsEntity<>(item.getBookmarkId().toString(), item)));
if (res.size() > 0) {
esUtil.insertBatch(EsConstant.BOOKMARK_INDEX, res);
}
index += size;
} while (res.size() == 500);
}
List<BookmarkEs> searchUserBookmark(int userId, String context);
}

69
bookMarkService/business/bookmark/src/main/java/com/fanxb/bookmark/business/bookmark/service/impl/BookmarkBackupServiceImpl.java Normal file
View File

@ -0,0 +1,69 @@
package com.fanxb.bookmark.business.bookmark.service.impl;
import com.fanxb.bookmark.business.bookmark.dao.BookmarkDao;
import com.fanxb.bookmark.business.bookmark.entity.BookmarkEs;
import com.fanxb.bookmark.business.bookmark.service.BookmarkBackupService;
import com.fanxb.bookmark.common.constant.EsConstant;
import com.fanxb.bookmark.common.entity.Bookmark;
import com.fanxb.bookmark.common.entity.EsEntity;
import com.fanxb.bookmark.common.util.EsUtil;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
/**
* Created with IntelliJ IDEA
* Created By Fxb
* Date: 2019/11/12
* Time: 0:22
*
* @author fanxb
*/
@Service
public class BookmarkBackupServiceImpl implements BookmarkBackupService {
@Autowired
private BookmarkDao bookmarkDao;
@Autowired
private EsUtil esUtil;
/**
* 一次同步BACKUP_SIZE条到es中
*/
private static final int BACKUP_SIZE = 500;
@Override
public void backupToEs() {
int start = 0;
List<Bookmark> list;
while ((list = bookmarkDao.getBookmarkListPage(BACKUP_SIZE, start)).size() != 0) {
List<EsEntity<BookmarkEs>> batchList = new ArrayList<>(list.size());
list.forEach(item -> batchList.add(new EsEntity<>(item.getBookmarkId().toString(), new BookmarkEs(item))));
esUtil.insertBatch(EsConstant.BOOKMARK_INDEX, batchList);
start += BACKUP_SIZE;
}
}
@Override
public void syncUserBookmark(int userId) {
//删除旧的数据
esUtil.deleteByQuery(EsConstant.BOOKMARK_INDEX, new TermQueryBuilder("userId", userId));
int index = 0;
int size = 500;
List<EsEntity<BookmarkEs>> res = new ArrayList<>();
do {
res.clear();
bookmarkDao.selectBookmarkEsByUserIdAndType(userId, 0, index, size)
.forEach(item -> res.add(new EsEntity<>(item.getBookmarkId().toString(), item)));
if (res.size() > 0) {
esUtil.insertBatch(EsConstant.BOOKMARK_INDEX, res);
}
index += size;
} while (res.size() == 500);
}
}

243
bookMarkService/business/bookmark/src/main/java/com/fanxb/bookmark/business/bookmark/service/impl/BookmarkServiceImpl.java Normal file
View File

@ -0,0 +1,243 @@
package com.fanxb.bookmark.business.bookmark.service.impl;
import com.fanxb.bookmark.business.bookmark.dao.BookmarkDao;
import com.fanxb.bookmark.business.bookmark.entity.BookmarkEs;
import com.fanxb.bookmark.business.bookmark.entity.MoveNodeBody;
import com.fanxb.bookmark.business.bookmark.service.BookmarkService;
import com.fanxb.bookmark.business.bookmark.service.PinYinService;
import com.fanxb.bookmark.common.constant.EsConstant;
import com.fanxb.bookmark.common.constant.RedisConstant;
import com.fanxb.bookmark.common.entity.Bookmark;
import com.fanxb.bookmark.common.entity.redis.UserBookmarkUpdate;
import com.fanxb.bookmark.common.util.EsUtil;
import com.fanxb.bookmark.common.util.RedisUtil;
import com.fanxb.bookmark.common.util.UserContextHolder;
import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.io.InputStream;
import java.util.*;
import java.util.stream.Collectors;
/**
* 类功能简述：
* 类功能详述：
*
* @author fanxb
* @date 2019/7/8 15:00
*/
@Service
@Slf4j
public class BookmarkServiceImpl implements BookmarkService {
@Autowired
private BookmarkDao bookmarkDao;
@Autowired
private StringRedisTemplate redisTemplate;
@Autowired
private PinYinService pinYinService;
@Autowired
private EsUtil esUtil;
@Override
@Transactional(rollbackFor = Exception.class)
public void parseBookmarkFile(int userId, InputStream stream, String path) throws Exception {
Document doc = Jsoup.parse(stream, "utf-8", "");
Elements elements = doc.select("html>body>dl>dt");
//获取当前层sort最大值
Integer sortBase = bookmarkDao.selectMaxSort(userId, path);
if (sortBase == null) {
sortBase = 0;
}
int count = 0;
List<Bookmark> bookmarks = new ArrayList<>();
for (int i = 0, length = elements.size(); i < length; i++) {
if (i == 0) {
Elements firstChildren = elements.get(0).child(1).children();
count = firstChildren.size();
for (int j = 0; j < count; j++) {
dealBookmark(userId, firstChildren.get(j), path, sortBase + j, bookmarks);
}
} else {
dealBookmark(userId, elements.get(i), path, sortBase + count + i - 1, bookmarks);
}
}
RedisUtil.addToMq(RedisConstant.BOOKMARK_UPDATE_TIME, new UserBookmarkUpdate(userId, System.currentTimeMillis()));
RedisUtil.addToMq(RedisConstant.BOOKMARK_INSERT_ES, bookmarks);
RedisUtil.addToMq(RedisConstant.BOOKMARK_PINYIN_CHANGE, bookmarks);
}
/**
* Description: 处理html节点，解析出文件夹和书签
*
* @param ele 待处理节点
* @param path 节点路径，不包含自身
* @param sort 当前层级中的排序序号
* @author fanxb
* @date 2019/7/8 14:49
*/
private void dealBookmark(int userId, Element ele, String path, int sort, List<Bookmark> bookmarks) {
if (!DT.equalsIgnoreCase(ele.tagName())) {
return;
}
Element first = ele.child(0);
if (A.equalsIgnoreCase(first.tagName())) {
//说明为链接
Bookmark node = new Bookmark(userId, path, first.ownText(), first.attr("href"), first.attr("icon")
, Long.parseLong(first.attr("add_date")) * 1000, sort);
//存入数据库
insertOne(node);
bookmarks.add(node);
} else {
//说明为文件夹
Bookmark node = new Bookmark(userId, path, first.ownText(), Long.parseLong(first.attr("add_date")) * 1000, sort);
Integer sortBase = 0;
//同名文件夹将会合并
if (insertOne(node)) {
sortBase = bookmarkDao.selectMaxSort(node.getUserId(), path);
if (sortBase == null) {
sortBase = 0;
}
}
String childPath = path + "." + node.getBookmarkId();
Elements children = ele.child(1).children();
for (int i = 0, size = children.size(); i < size; i++) {
dealBookmark(userId, children.get(i), childPath, sortBase + i + 1, bookmarks);
}
}
}
/**
* Description: 插入一条书签，如果已经存在同名书签将跳过
*
* @param node node
* @return boolean 如果已经存在返回true，否则false
* @author fanxb
* @date 2019/7/8 17:25
*/
private boolean insertOne(Bookmark node) {
//先根据name,userId,parentId获取此节点id
Integer id = bookmarkDao.selectIdByUserIdAndNameAndPath(node.getUserId(), node.getName(), node.getPath());
if (id == null) {
bookmarkDao.insertOne(node);
return false;
} else {
node.setBookmarkId(id);
return true;
}
}
@Override
public Map<String, List<Bookmark>> getOneBookmarkTree(int userId) {
List<Bookmark> list = bookmarkDao.getListByUserId(userId);
Map<String, List<Bookmark>> map = new HashMap<>(50);
list.forEach(item -> {
map.computeIfAbsent(item.getPath(), k -> new ArrayList<>());
map.get(item.getPath()).add(item);
});
return map;
}
@Override
public List<Bookmark> getBookmarkListByPath(int userId, String path) {
return bookmarkDao.getListByUserIdAndPath(userId, path);
}
@Override
@Transactional(rollbackFor = Exception.class)
public void batchDelete(int userId, List<Integer> folderIdList, List<Integer> bookmarkIdList) {
Set<String> set = new HashSet<>();
for (Integer item : folderIdList) {
set.addAll(bookmarkDao.getChildrenBookmarkId(userId, item).stream().map(String::valueOf).collect(Collectors.toSet()));
bookmarkDao.deleteUserFolder(userId, item);
bookmarkIdList.add(item);
}
if (bookmarkIdList.size() > 0) {
bookmarkDao.deleteUserBookmark(userId, bookmarkIdList);
}
set.addAll(bookmarkIdList.stream().map(String::valueOf).collect(Collectors.toSet()));
RedisUtil.addToMq(RedisConstant.BOOKMARK_UPDATE_TIME, new UserBookmarkUpdate(userId, System.currentTimeMillis()));
RedisUtil.addToMq(RedisConstant.BOOKMARK_DELETE_ES, set);
}
@Override
@Transactional(rollbackFor = Exception.class)
public Bookmark addOne(Bookmark bookmark) {
int userId = UserContextHolder.get().getUserId();
Integer sort = bookmarkDao.selectMaxSort(userId, bookmark.getPath());
bookmark.setSort(sort == null ? 1 : sort + 1);
bookmark.setUserId(userId);
bookmark.setCreateTime(System.currentTimeMillis());
bookmark.setAddTime(bookmark.getCreateTime());
bookmark.setSearchKey(pinYinService.changeString(bookmark.getName()));
bookmarkDao.insertOne(bookmark);
//如果是书签，插入到es中
if (bookmark.getType() == 0) {
RedisUtil.addToMq(RedisConstant.BOOKMARK_INSERT_ES, Collections.singleton(bookmark));
}
RedisUtil.addToMq(RedisConstant.BOOKMARK_UPDATE_TIME, new UserBookmarkUpdate(userId, System.currentTimeMillis()));
return bookmark;
}
@Override
@Transactional(rollbackFor = Exception.class)
public void updateOne(int userId, Bookmark bookmark) {
bookmark.setUserId(userId);
bookmark.setSearchKey(pinYinService.changeString(bookmark.getName()));
bookmarkDao.editBookmark(bookmark);
if (bookmark.getType() == 0) {
RedisUtil.addToMq(RedisConstant.BOOKMARK_INSERT_ES, Collections.singleton(bookmark));
}
RedisUtil.addToMq(RedisConstant.BOOKMARK_UPDATE_TIME, new UserBookmarkUpdate(userId, System.currentTimeMillis()));
}
@Override
@Transactional(rollbackFor = Exception.class)
public void moveNode(int userId, MoveNodeBody body) {
if (body.getSort() == -1) {
Integer max = bookmarkDao.selectMaxSort(userId, body.getTargetPath());
body.setSort(max == null ? 1 : max);
} else {
//更新目标节点的sort
bookmarkDao.sortPlus(userId, body.getTargetPath(), body.getSort());
}
//如果目标位置和当前位置不在一个层级中需要更新子节点的path
if (!body.getTargetPath().equals(body.getSourcePath())) {
bookmarkDao.updateChildrenPath(userId, body.getSourcePath() + "." + body.getBookmarkId()
, body.getTargetPath() + "." + body.getBookmarkId());
}
//更新被移动节点的path和sort
bookmarkDao.updatePathAndSort(userId, body.getBookmarkId(), body.getTargetPath(), body.getSort());
RedisUtil.addToMq(RedisConstant.BOOKMARK_UPDATE_TIME, new UserBookmarkUpdate(userId, System.currentTimeMillis()));
}
@Override
public List<BookmarkEs> searchUserBookmark(int userId, String context) {
BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
boolQueryBuilder.must(QueryBuilders.termQuery("userId", userId));
boolQueryBuilder.must(QueryBuilders.multiMatchQuery(context, "name", "url"));
SearchSourceBuilder builder = new SearchSourceBuilder();
builder.size(5);
builder.query(boolQueryBuilder);
return esUtil.search(EsConstant.BOOKMARK_INDEX, builder, BookmarkEs.class);
}
}

8
bookMarkService/business/bookmark/src/main/java/com/fanxb/bookmark/business/bookmark/service/impl/PinYinServiceImpl.java
View File

@ -1,15 +1,17 @@
package com.fanxb.bookmark.business.bookmark.service.impl;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.util.StrUtil;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.fanxb.bookmark.business.bookmark.dao.BookmarkDao;
import com.fanxb.bookmark.business.bookmark.entity.PinYinBody;
import com.fanxb.bookmark.business.bookmark.service.PinYinService;
import com.fanxb.bookmark.common.constant.Constant;
import com.fanxb.bookmark.common.constant.RedisConstant;
import com.fanxb.bookmark.common.entity.Bookmark;
import com.fanxb.bookmark.common.entity.redis.UserBookmarkUpdate;
import com.fanxb.bookmark.common.util.HttpUtil;
import com.fanxb.bookmark.common.util.RedisUtil;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@ -54,10 +56,12 @@ public class PinYinServiceImpl implements PinYinService {
bookmarkDao.updateSearchKey(bookmarks.get(j).getBookmarkId(), resList.get(j));
}
if (bookmarks.size() < SIZE) {
return;
break;
}
i = bookmarks.get(SIZE - 1).getBookmarkId();
}
//更新所有用户的上次刷新时间
RedisUtil.addToMq(RedisConstant.BOOKMARK_UPDATE_TIME, new UserBookmarkUpdate(-1, System.currentTimeMillis()));
}
@Override

15
bookMarkService/common/src/main/java/com/fanxb/bookmark/common/configuration/MqConfiguration.java
View File

@ -1,9 +1,7 @@
package com.fanxb.bookmark.common.configuration;
import cn.hutool.core.util.StrUtil;
import com.fanxb.bookmark.common.annotation.MqConsumer;
import com.fanxb.bookmark.common.entity.redis.RedisConsumer;
import com.fanxb.bookmark.common.exception.CustomException;
import com.fanxb.bookmark.common.factory.ThreadPoolFactory;
import com.fanxb.bookmark.common.util.RedisUtil;
import lombok.extern.slf4j.Slf4j;
@ -13,14 +11,12 @@ import org.springframework.boot.ApplicationRunner;
import org.springframework.context.ApplicationContext;
import org.springframework.stereotype.Component;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
/**
@ -57,7 +53,7 @@ public class MqConfiguration implements ApplicationRunner {
MqConsumer annotation = annotations[0];
topicMap.computeIfAbsent(annotation.value(), k -> new ArrayList<>()).add((RedisConsumer) item);
});
log.info("es订阅信息汇总完毕！！！！！！");
log.info("redis订阅信息汇总完毕！！！！！！");
//由一个线程始终循环获取es队列数据
threadPoolExecutor.execute(loop());
}
@ -114,14 +110,5 @@ public class MqConfiguration implements ApplicationRunner {
});
}
}
}
@MqConsumer("test1212")
class Test implements RedisConsumer {
@Override
public void deal(String message) {
System.out.println(message);
}
}

12
bookMarkService/common/src/main/java/com/fanxb/bookmark/common/constant/RedisConstant.java
View File

@ -15,4 +15,16 @@ public class RedisConstant {
* 某用户书签数据更新时间,该队列左进右出
*/
public static final String BOOKMARK_UPDATE_TIME = "bookmark_update_time";
/**
* 某个用户上传了文件夹，需要进行书签转化
*/
public static final String BOOKMARK_PINYIN_CHANGE = "bookmark_pinyin_change";
/**
* 插入书签数据到es中
*/
public static final String BOOKMARK_INSERT_ES = "bookmark_insert_es";
/**
* 从es中删除数据
*/
public static final String BOOKMARK_DELETE_ES = "bookmark_DELETE_es";
}

5
bookMarkService/common/src/main/java/com/fanxb/bookmark/common/entity/redis/UserBookmarkUpdate.java
View File

@ -20,6 +20,11 @@ public class UserBookmarkUpdate {
*/
private long updateTime;
public UserBookmarkUpdate(int userId) {
this.userId = userId;
this.updateTime = System.currentTimeMillis();
}
@Override
public String toString() {
return JSON.toJSONString(this);

5
bookMarkService/common/src/main/java/com/fanxb/bookmark/common/util/EsUtil.java
View File

@ -2,7 +2,6 @@ package com.fanxb.bookmark.common.util;
import com.alibaba.fastjson.JSON;
import com.fanxb.bookmark.common.constant.EsConstant;
import com.fanxb.bookmark.common.entity.Bookmark;
import com.fanxb.bookmark.common.entity.EsEntity;
import com.fanxb.bookmark.common.exception.CustomException;
import com.fanxb.bookmark.common.exception.EsException;
@ -142,9 +141,9 @@ public class EsUtil {
* @author fanxb
* @date 2019/7/25 14:24
*/
public <T> void deleteBatch(String index, Collection<T> idList) {
public <T> void deleteBatch(String index, Collection<String> idList) {
BulkRequest request = new BulkRequest();
idList.forEach(item -> request.add(new DeleteRequest(index, item.toString())));
idList.forEach(item -> request.add(new DeleteRequest(index, item)));
try {
client.bulk(request, RequestOptions.DEFAULT);
} catch (Exception e) {

2
front/src/pages/manage/OverView/index.jsx
View File

@ -56,7 +56,7 @@ class OverView extends React.Component {
async componentWillMount() {
await httpUtil.get("/user/loginStatus");
this.state.timer = setInterval(this.checkCache.bind(this), 5 * 60 * 1000);
this.state.timer = setInterval(this.checkCache.bind(this), 10000);
setTimeout(this.checkCache.bind(this), 5000);
this.props.refresh();
await cacheBookmarkData();

20
front/src/util/cacheUtil.js
View File

@ -1,10 +1,11 @@
/* eslint-disable no-undef */
import httpUtil from "./httpUtil";
import config from "./config";
/**
* 缓存工具类
* web版本
*/
export const WEB_VERSION = "webVersion";
/**
* 全部书签数据key
*/
@ -24,7 +25,9 @@ export const TREE_LIST_USER_ID = "treeListDataUserId";
export async function cacheBookmarkData() {
let currentId = JSON.parse(window.atob(window.token.split(".")[1])).userId;
let cacheId = await localforage.getItem(TREE_LIST_USER_ID);
if (currentId && currentId !== cacheId) {
let webVersion = await localforage.getItem(WEB_VERSION);
if ((currentId && currentId !== cacheId) || config.version !== webVersion) {
await localforage.setItem(WEB_VERSION, config.version);
await clearCache();
}
let res = await localforage.getItem(TREE_LIST_KEY);
@ -213,16 +216,7 @@ export async function keySearch(content) {
if (item.type === 1) {
continue;
}
if (!item.lowName) {
item.lowName = item.name + "////" + item.name.toLocaleLowerCase();
}
if (!item.lowUrl) {
item.lowUrl = item.url + "////" + item.url.toLocaleLowerCase();
}
if (
item.lowName.indexOf(content) > -1 ||
item.lowUrl.indexOf(content) > -1
) {
if (item.searchKey.indexOf(content) > -1) {
res.push(item);
if (res.length >= 12) {
console.info("搜索耗时：" + (Date.now() - time1));