diff --git a/bookMarkService/business/bookmark/src/main/java/com/fanxb/bookmark/business/bookmark/constant/FileConstant.java b/bookMarkService/business/bookmark/src/main/java/com/fanxb/bookmark/business/bookmark/constant/FileConstant.java new file mode 100644 index 0000000..0e9f78a --- /dev/null +++ b/bookMarkService/business/bookmark/src/main/java/com/fanxb/bookmark/business/bookmark/constant/FileConstant.java @@ -0,0 +1,16 @@ +package com.fanxb.bookmark.business.bookmark.constant; + +import java.nio.file.Paths; + +/** + * TODO + * + * @author fanxb + */ +public class FileConstant { + + /** + * 网站icon存储路径 + */ + public static final String FAVICON_PATH = Paths.get("files", "public", "favicon").toString(); +} diff --git a/bookMarkService/business/bookmark/src/main/java/com/fanxb/bookmark/business/bookmark/dao/HostIconDao.java b/bookMarkService/business/bookmark/src/main/java/com/fanxb/bookmark/business/bookmark/dao/HostIconDao.java new file mode 100644 index 0000000..915100b --- /dev/null +++ b/bookMarkService/business/bookmark/src/main/java/com/fanxb/bookmark/business/bookmark/dao/HostIconDao.java @@ -0,0 +1,33 @@ +package com.fanxb.bookmark.business.bookmark.dao; + +import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; + +/** + * @author fanxb + */ +@Mapper +public interface HostIconDao { + + /** + * 插入一条数据 + * + * @param host host + * @param iconPath path + * @author fanxb + */ + @Insert("insert into host_icon(host,iconPath) value(#{host},#{iconPath})") + void insert(@Param("host") String host, @Param("iconPath") String iconPath); + + /** + * 根据host获取iconPath + * + * @param host host + * @return {@link String} + * @author fanxb + */ + @Select("select iconPath from host_icon where host=#{host}") + String selectByHost(String host); +} diff --git a/bookMarkService/business/bookmark/src/main/java/com/fanxb/bookmark/business/bookmark/service/impl/BookmarkServiceImpl.java b/bookMarkService/business/bookmark/src/main/java/com/fanxb/bookmark/business/bookmark/service/impl/BookmarkServiceImpl.java index d7e794f..6e50bc6 100644 --- a/bookMarkService/business/bookmark/src/main/java/com/fanxb/bookmark/business/bookmark/service/impl/BookmarkServiceImpl.java +++ b/bookMarkService/business/bookmark/src/main/java/com/fanxb/bookmark/business/bookmark/service/impl/BookmarkServiceImpl.java @@ -1,21 +1,29 @@ package com.fanxb.bookmark.business.bookmark.service.impl; +import cn.hutool.core.io.FileUtil; import cn.hutool.core.util.ArrayUtil; +import cn.hutool.core.util.CharsetUtil; import cn.hutool.core.util.StrUtil; import com.alibaba.fastjson.JSON; import com.fanxb.bookmark.business.api.UserApi; +import com.fanxb.bookmark.business.bookmark.constant.FileConstant; import com.fanxb.bookmark.business.bookmark.dao.BookmarkDao; +import com.fanxb.bookmark.business.bookmark.dao.HostIconDao; import com.fanxb.bookmark.business.bookmark.entity.BookmarkEs; import com.fanxb.bookmark.business.bookmark.entity.MoveNodeBody; import com.fanxb.bookmark.business.bookmark.entity.redis.BookmarkDeleteMessage; import com.fanxb.bookmark.business.bookmark.entity.redis.VisitNumPlus; import com.fanxb.bookmark.business.bookmark.service.BookmarkService; import com.fanxb.bookmark.business.bookmark.service.PinYinService; +import com.fanxb.bookmark.common.constant.CommonConstant; import com.fanxb.bookmark.common.constant.EsConstant; import com.fanxb.bookmark.common.constant.RedisConstant; import com.fanxb.bookmark.common.entity.po.Bookmark; +import com.fanxb.bookmark.common.exception.CustomException; import com.fanxb.bookmark.common.util.*; import lombok.extern.slf4j.Slf4j; +import okhttp3.Request; +import okhttp3.Response; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -31,6 +39,10 @@ import org.springframework.transaction.annotation.Transactional; import java.io.InputStream; import java.net.MalformedURLException; import java.net.URL; +import java.net.URLEncoder; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; import java.util.*; import java.util.stream.Collectors; @@ -51,13 +63,15 @@ public class BookmarkServiceImpl implements BookmarkService { private final PinYinService pinYinService; private final UserApi userApi; private final EsUtil esUtil; + private final HostIconDao hostIconDao; @Autowired - public BookmarkServiceImpl(BookmarkDao bookmarkDao, PinYinService pinYinService, UserApi userApi, EsUtil esUtil) { + public BookmarkServiceImpl(BookmarkDao bookmarkDao, PinYinService pinYinService, UserApi userApi, EsUtil esUtil, HostIconDao hostIconDao) { this.bookmarkDao = bookmarkDao; this.pinYinService = pinYinService; this.userApi = userApi; this.esUtil = esUtil; + this.hostIconDao = hostIconDao; } @Override @@ -201,9 +215,7 @@ public class BookmarkServiceImpl implements BookmarkService { bookmark.setUserId(userId); bookmark.setCreateTime(System.currentTimeMillis()); bookmark.setAddTime(bookmark.getCreateTime()); - if (bookmark.getIcon() == null) { - bookmark.setIcon(getIconBase64(bookmark.getUrl())); - } + bookmark.setIcon(getIconPath(bookmark.getUrl())); //文件夹和书签都建立搜索key pinYinService.changeBookmark(bookmark); bookmarkDao.insertOne(bookmark); @@ -217,7 +229,7 @@ public class BookmarkServiceImpl implements BookmarkService { bookmark.setUserId(userId); if (bookmark.getType() == 0) { pinYinService.changeBookmark(bookmark); - bookmark.setIcon(getIconBase64(bookmark.getUrl())); + bookmark.setIcon(getIconPath(bookmark.getUrl())); } bookmarkDao.editBookmark(bookmark); userApi.versionPlus(userId); @@ -276,7 +288,7 @@ public class BookmarkServiceImpl implements BookmarkService { while ((deal = bookmarkDao.selectUserNoIcon(userId, start, size)).size() > 0) { start += size; deal.forEach(item -> { - String icon = getIconBase64(item.getUrl()); + String icon = getIconPath(item.getUrl()); if (StrUtil.isNotEmpty(icon)) { bookmarkDao.updateIcon(item.getBookmarkId(), icon); } @@ -307,21 +319,60 @@ public class BookmarkServiceImpl implements BookmarkService { return resPath; } - private String getIconBase64(String url) { + /** + * 获取icon + * + * @param url url + * @return {@link String} + * @throws + * @author fanxb + */ + private String getIconPath(String url) { if (StrUtil.isEmpty(url)) { return ""; } + String host; try { URL urlObj = new URL(url); - byte[] data = HttpUtil.download(urlIconAddress + "/icon?url=" + urlObj.getHost() + "&size=8..16..64", false); - String base64 = new String(Base64.getEncoder().encode(data)); - if (StrUtil.isNotEmpty(base64)) { - return "data:image/png;base64," + base64; - } else { - log.warn("url无法获取icon:{}", url); - } - } catch (MalformedURLException e) { + host = urlObj.getHost(); + } catch (Exception e) { log.warn("url无法解析出domain:{}", url); + return ""; + } + String iconPath = hostIconDao.selectByHost(host); + if (iconPath != null) { + return iconPath; + } + iconPath = saveFile(host, "http://" + host + "/favicon.ico"); + if (StrUtil.isEmpty(iconPath)) { + iconPath = saveFile(host, urlIconAddress + "/icon?url=" + host + "&size=16..64..256"); + } + if (StrUtil.isNotEmpty(iconPath)) { + hostIconDao.insert(host, iconPath); + } + return iconPath; + } + + private String saveFile(String host, String url) { + try { + try (Response res = HttpUtil.getClient(false).newCall(new Request.Builder().url(url) + .header("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.75 Safari/537.36 Edg/100.0.1185.36") + .get().build()).execute()) { + assert res.body() != null; + if (!HttpUtil.checkIsOk(res.code())) { + throw new CustomException("请求错误:" + res.code()); + } + byte[] data = res.body().byteStream().readAllBytes(); + if (data.length > 0) { + String iconUrl = res.request().url().toString(); + String fileName = URLEncoder.encode(host, StandardCharsets.UTF_8) + iconUrl.substring(iconUrl.lastIndexOf(".")); + String filePath = Paths.get(CommonConstant.fileSavePath, FileConstant.FAVICON_PATH, host.substring(0, 2), fileName).toString(); + FileUtil.writeBytes(data, filePath); + return filePath; + } else { + log.info("未获取到icon:{}", url); + } + } } catch (Exception e) { log.error("url获取icon故障:{}", url, e); } diff --git a/bookMarkService/business/user/src/main/java/com/fanxb/bookmark/business/user/constant/FileConstant.java b/bookMarkService/business/user/src/main/java/com/fanxb/bookmark/business/user/constant/FileConstant.java index eccc028..9f0ac6f 100644 --- a/bookMarkService/business/user/src/main/java/com/fanxb/bookmark/business/user/constant/FileConstant.java +++ b/bookMarkService/business/user/src/main/java/com/fanxb/bookmark/business/user/constant/FileConstant.java @@ -1,5 +1,6 @@ package com.fanxb.bookmark.business.user.constant; +import com.fanxb.bookmark.common.constant.CommonConstant; import org.springframework.stereotype.Component; import java.nio.file.Paths; @@ -17,6 +18,6 @@ public class FileConstant { /** * 用户头像目录 */ - public static String iconPath = Paths.get("files", "public", "icon").toString(); + public static String iconPath = Paths.get(CommonConstant.fileSavePath, "files", "public", "icon").toString(); } diff --git a/bookMarkService/common/src/main/java/com/fanxb/bookmark/common/util/HttpUtil.java b/bookMarkService/common/src/main/java/com/fanxb/bookmark/common/util/HttpUtil.java index ffc946f..17665d4 100644 --- a/bookMarkService/common/src/main/java/com/fanxb/bookmark/common/util/HttpUtil.java +++ b/bookMarkService/common/src/main/java/com/fanxb/bookmark/common/util/HttpUtil.java @@ -12,8 +12,6 @@ import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; import javax.servlet.http.HttpServletRequest; -import java.io.InputStream; -import java.io.OutputStream; import java.net.InetSocketAddress; import java.net.Proxy; import java.util.Map; @@ -51,42 +49,33 @@ public class HttpUtil { /** * 无代理环境 */ - private static final OkHttpClient CLIENT = new OkHttpClient.Builder().connectTimeout(2, TimeUnit.SECONDS) + private static final OkHttpClient CLIENT = new OkHttpClient.Builder().connectTimeout(1, TimeUnit.SECONDS) .readTimeout(60, TimeUnit.SECONDS) .build(); + + /** + * 获取客户端 + * + * @param proxy 是否代理 + * @return {@link OkHttpClient} + * @author fanxb + */ + public static OkHttpClient getClient(boolean proxy) { + return proxy ? PROXY_CLIENT : CLIENT; + } + public static final MediaType JSON = MediaType.get("application/json; charset=utf-8"); @PostConstruct public void init() { - OkHttpClient.Builder builder = new OkHttpClient.Builder(); + OkHttpClient.Builder builder = new OkHttpClient.Builder().connectTimeout(1, TimeUnit.SECONDS).readTimeout(60, TimeUnit.SECONDS); log.info("代理配置，ip:{},port:{}", proxyIp, proxyPort); if (StrUtil.isNotBlank(proxyIp) && StrUtil.isNotBlank(proxyPort)) { builder.proxy(new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyIp, Integer.parseInt(proxyPort)))); proxyExist = true; - } - PROXY_CLIENT = builder.connectTimeout(10, TimeUnit.SECONDS) - .readTimeout(60, TimeUnit.SECONDS) - .build(); - } - - /*** - * 下载文件 - * @author fanxb - * @param url 下载链接 - * @param proxy 是否使用代理 - * @return java.io.InputStream - * @date 2021/3/12 - **/ - public static byte[] download(String url, boolean proxy) { - try (Response res = (proxy ? PROXY_CLIENT : CLIENT).newCall(new Request.Builder().url(url).build()).execute()) { - assert res.body() != null; - if (checkIsOk(res.code())) { - return res.body().byteStream().readAllBytes(); - } else { - throw new CustomException("下载出现问题:" + res.body().string()); - } - } catch (Exception e) { - throw new CustomException(e); + PROXY_CLIENT = builder.build(); + } else { + PROXY_CLIENT = CLIENT; } } @@ -270,6 +259,8 @@ public class HttpUtil { } return ipAddress; } + + } diff --git a/bookMarkService/files/public/icon/1.1614853232994.html b/bookMarkService/files/public/icon/1.1614853232994.html deleted file mode 100644 index 909688a..0000000 --- a/bookMarkService/files/public/icon/1.1614853232994.html +++ /dev/null @@ -1,17 +0,0 @@ - - - -Bookmarks -

Bookmarks Menu

- -

-

1 -
2 -

f1

-

-

f11 -
f12 -

-

diff --git a/bookMarkService/files/public/icon/1.1614853330412.html b/bookMarkService/files/public/icon/1.1614853330412.html deleted file mode 100644 index 909688a..0000000 --- a/bookMarkService/files/public/icon/1.1614853330412.html +++ /dev/null @@ -1,17 +0,0 @@ - - - -Bookmarks -

Bookmarks Menu

- -

-

1 -
2 -

f1

-

-

f11 -
f12 -

-

diff --git a/bookMarkService/web/src/main/resources/db/migration/V21__新增域名path表.sql b/bookMarkService/web/src/main/resources/db/migration/V21__新增域名path表.sql new file mode 100644 index 0000000..3f3fdd0 --- /dev/null +++ b/bookMarkService/web/src/main/resources/db/migration/V21__新增域名path表.sql @@ -0,0 +1,10 @@ +CREATE TABLE bookmark.host_icon ( + id INT UNSIGNED auto_increment NOT NULL, + host varchar(300) NOT NULL COMMENT 'host', + iconPath varchar(330) NOT NULL, + CONSTRAINT host_icon_pk PRIMARY KEY (id) +) +ENGINE=InnoDB +DEFAULT CHARSET=utf8mb4 +COLLATE=utf8mb4_0900_ai_ci; +CREATE INDEX host_icon_host_IDX USING BTREE ON bookmark.host_icon (host(20)); diff --git a/bookmark_front/src/views/noHead/addBookmark/index.vue b/bookmark_front/src/views/noHead/addBookmark/index.vue index 5961dfa..1e9c1bc 100644 --- a/bookmark_front/src/views/noHead/addBookmark/index.vue +++ b/bookmark_front/src/views/noHead/addBookmark/index.vue @@ -1,5 +1,8 @@