temp

bookMarkService/business/bookmark/src/main/java/com/fanxb/bookmark/business/bookmark/constant/FileConstant.java

@@ -0,0 +1,16 @@
+package com.fanxb.bookmark.business.bookmark.constant;
+
+import java.nio.file.Paths;
+
+/**
+ * TODO
+ *
+ * @author fanxb
+ */
+public class FileConstant {
+
+    /**
+     * 网站icon存储路径
+     */
+    public static final String FAVICON_PATH = Paths.get("files", "public", "favicon").toString();
+}

bookMarkService/business/bookmark/src/main/java/com/fanxb/bookmark/business/bookmark/dao/HostIconDao.java

@@ -0,0 +1,33 @@
+package com.fanxb.bookmark.business.bookmark.dao;
+
+import org.apache.ibatis.annotations.Insert;
+import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
+import org.apache.ibatis.annotations.Select;
+
+/**
+ * @author fanxb
+ */
+@Mapper
+public interface HostIconDao {
+
+    /**
+     * 插入一条数据
+     *
+     * @param host     host
+     * @param iconPath path
+     * @author fanxb
+     */
+    @Insert("insert into host_icon(host,iconPath) value(#{host},#{iconPath})")
+    void insert(@Param("host") String host, @Param("iconPath") String iconPath);
+
+    /**
+     * 根据host获取iconPath
+     *
+     * @param host host
+     * @return {@link String}
+     * @author fanxb
+     */
+    @Select("select iconPath from host_icon where host=#{host}")
+    String selectByHost(String host);
+}

bookMarkService/business/bookmark/src/main/java/com/fanxb/bookmark/business/bookmark/service/impl/BookmarkServiceImpl.java

@@ -1,21 +1,29 @@
 package com.fanxb.bookmark.business.bookmark.service.impl;
 
+import cn.hutool.core.io.FileUtil;
 import cn.hutool.core.util.ArrayUtil;
+import cn.hutool.core.util.CharsetUtil;
 import cn.hutool.core.util.StrUtil;
 import com.alibaba.fastjson.JSON;
 import com.fanxb.bookmark.business.api.UserApi;
+import com.fanxb.bookmark.business.bookmark.constant.FileConstant;
 import com.fanxb.bookmark.business.bookmark.dao.BookmarkDao;
+import com.fanxb.bookmark.business.bookmark.dao.HostIconDao;
 import com.fanxb.bookmark.business.bookmark.entity.BookmarkEs;
 import com.fanxb.bookmark.business.bookmark.entity.MoveNodeBody;
 import com.fanxb.bookmark.business.bookmark.entity.redis.BookmarkDeleteMessage;
 import com.fanxb.bookmark.business.bookmark.entity.redis.VisitNumPlus;
 import com.fanxb.bookmark.business.bookmark.service.BookmarkService;
 import com.fanxb.bookmark.business.bookmark.service.PinYinService;
+import com.fanxb.bookmark.common.constant.CommonConstant;
 import com.fanxb.bookmark.common.constant.EsConstant;
 import com.fanxb.bookmark.common.constant.RedisConstant;
 import com.fanxb.bookmark.common.entity.po.Bookmark;
+import com.fanxb.bookmark.common.exception.CustomException;
 import com.fanxb.bookmark.common.util.*;
 import lombok.extern.slf4j.Slf4j;
+import okhttp3.Request;
+import okhttp3.Response;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
@@ -31,6 +39,10 @@ import org.springframework.transaction.annotation.Transactional;
 import java.io.InputStream;
 import java.net.MalformedURLException;
 import java.net.URL;
+import java.net.URLEncoder;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Paths;
 import java.util.*;
 import java.util.stream.Collectors;
 
@@ -51,13 +63,15 @@ public class BookmarkServiceImpl implements BookmarkService {
     private final PinYinService pinYinService;
     private final UserApi userApi;
     private final EsUtil esUtil;
+    private final HostIconDao hostIconDao;
 
     @Autowired
-    public BookmarkServiceImpl(BookmarkDao bookmarkDao, PinYinService pinYinService, UserApi userApi, EsUtil esUtil) {
+    public BookmarkServiceImpl(BookmarkDao bookmarkDao, PinYinService pinYinService, UserApi userApi, EsUtil esUtil, HostIconDao hostIconDao) {
         this.bookmarkDao = bookmarkDao;
         this.pinYinService = pinYinService;
         this.userApi = userApi;
         this.esUtil = esUtil;
+        this.hostIconDao = hostIconDao;
     }
 
     @Override
@@ -201,9 +215,7 @@ public class BookmarkServiceImpl implements BookmarkService {
         bookmark.setUserId(userId);
         bookmark.setCreateTime(System.currentTimeMillis());
         bookmark.setAddTime(bookmark.getCreateTime());
-        if (bookmark.getIcon() == null) {
+        bookmark.setIcon(getIconPath(bookmark.getUrl()));
-            bookmark.setIcon(getIconBase64(bookmark.getUrl()));
-        }
         //文件夹和书签都建立搜索key
         pinYinService.changeBookmark(bookmark);
         bookmarkDao.insertOne(bookmark);
@@ -217,7 +229,7 @@ public class BookmarkServiceImpl implements BookmarkService {
         bookmark.setUserId(userId);
         if (bookmark.getType() == 0) {
             pinYinService.changeBookmark(bookmark);
-            bookmark.setIcon(getIconBase64(bookmark.getUrl()));
+            bookmark.setIcon(getIconPath(bookmark.getUrl()));
         }
         bookmarkDao.editBookmark(bookmark);
         userApi.versionPlus(userId);
@@ -276,7 +288,7 @@ public class BookmarkServiceImpl implements BookmarkService {
         while ((deal = bookmarkDao.selectUserNoIcon(userId, start, size)).size() > 0) {
             start += size;
             deal.forEach(item -> {
-                String icon = getIconBase64(item.getUrl());
+                String icon = getIconPath(item.getUrl());
                 if (StrUtil.isNotEmpty(icon)) {
                     bookmarkDao.updateIcon(item.getBookmarkId(), icon);
                 }
@@ -307,21 +319,60 @@ public class BookmarkServiceImpl implements BookmarkService {
         return resPath;
     }
 
-    private String getIconBase64(String url) {
+    /**
+     * 获取icon
+     *
+     * @param url url
+     * @return {@link String}
+     * @throws
+     * @author fanxb
+     */
+    private String getIconPath(String url) {
         if (StrUtil.isEmpty(url)) {
             return "";
         }
+        String host;
         try {
             URL urlObj = new URL(url);
-            byte[] data = HttpUtil.download(urlIconAddress + "/icon?url=" + urlObj.getHost() + "&size=8..16..64", false);
+            host = urlObj.getHost();
-            String base64 = new String(Base64.getEncoder().encode(data));
+        } catch (Exception e) {
-            if (StrUtil.isNotEmpty(base64)) {
-                return "data:image/png;base64," + base64;
-            } else {
-                log.warn("url无法获取icon:{}", url);
-            }
-        } catch (MalformedURLException e) {
             log.warn("url无法解析出domain:{}", url);
+            return "";
+        }
+        String iconPath = hostIconDao.selectByHost(host);
+        if (iconPath != null) {
+            return iconPath;
+        }
+        iconPath = saveFile(host, "http://" + host + "/favicon.ico");
+        if (StrUtil.isEmpty(iconPath)) {
+            iconPath = saveFile(host, urlIconAddress + "/icon?url=" + host + "&size=16..64..256");
+        }
+        if (StrUtil.isNotEmpty(iconPath)) {
+            hostIconDao.insert(host, iconPath);
+        }
+        return iconPath;
+    }
+
+    private String saveFile(String host, String url) {
+        try {
+            try (Response res = HttpUtil.getClient(false).newCall(new Request.Builder().url(url)
+                    .header("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.75 Safari/537.36 Edg/100.0.1185.36")
+                    .get().build()).execute()) {
+                assert res.body() != null;
+                if (!HttpUtil.checkIsOk(res.code())) {
+                    throw new CustomException("请求错误:" + res.code());
+                }
+                byte[] data = res.body().byteStream().readAllBytes();
+                if (data.length > 0) {
+                    String iconUrl = res.request().url().toString();
+                    String fileName = URLEncoder.encode(host, StandardCharsets.UTF_8) + iconUrl.substring(iconUrl.lastIndexOf("."));
+                    String filePath = Paths.get(CommonConstant.fileSavePath, FileConstant.FAVICON_PATH, host.substring(0, 2), fileName).toString();
+                    FileUtil.writeBytes(data, filePath);
+                    return filePath;
+                } else {
+                    log.info("未获取到icon:{}", url);
+                }
+            }
         } catch (Exception e) {
             log.error("url获取icon故障:{}", url, e);
         }

bookMarkService/business/user/src/main/java/com/fanxb/bookmark/business/user/constant/FileConstant.java

@@ -1,5 +1,6 @@
 package com.fanxb.bookmark.business.user.constant;
 
+import com.fanxb.bookmark.common.constant.CommonConstant;
 import org.springframework.stereotype.Component;
 
 import java.nio.file.Paths;
@@ -17,6 +18,6 @@ public class FileConstant {
     /**
      * 用户头像目录
      */
-    public static String iconPath = Paths.get("files", "public", "icon").toString();
+    public static String iconPath = Paths.get(CommonConstant.fileSavePath, "files", "public", "icon").toString();
 
 }

bookMarkService/common/src/main/java/com/fanxb/bookmark/common/util/HttpUtil.java

@@ -12,8 +12,6 @@ import org.springframework.stereotype.Component;
 
 import javax.annotation.PostConstruct;
 import javax.servlet.http.HttpServletRequest;
-import java.io.InputStream;
-import java.io.OutputStream;
 import java.net.InetSocketAddress;
 import java.net.Proxy;
 import java.util.Map;
@@ -51,42 +49,33 @@ public class HttpUtil {
     /**
      * 无代理环境
      */
-    private static final OkHttpClient CLIENT = new OkHttpClient.Builder().connectTimeout(2, TimeUnit.SECONDS)
+    private static final OkHttpClient CLIENT = new OkHttpClient.Builder().connectTimeout(1, TimeUnit.SECONDS)
             .readTimeout(60, TimeUnit.SECONDS)
             .build();
+
+    /**
+     * 获取客户端
+     *
+     * @param proxy 是否代理
+     * @return {@link OkHttpClient}
+     * @author fanxb
+     */
+    public static OkHttpClient getClient(boolean proxy) {
+        return proxy ? PROXY_CLIENT : CLIENT;
+    }
+
     public static final MediaType JSON = MediaType.get("application/json; charset=utf-8");
 
     @PostConstruct
     public void init() {
-        OkHttpClient.Builder builder = new OkHttpClient.Builder();
+        OkHttpClient.Builder builder = new OkHttpClient.Builder().connectTimeout(1, TimeUnit.SECONDS).readTimeout(60, TimeUnit.SECONDS);
         log.info("代理配置，ip:{},port:{}", proxyIp, proxyPort);
         if (StrUtil.isNotBlank(proxyIp) && StrUtil.isNotBlank(proxyPort)) {
             builder.proxy(new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyIp, Integer.parseInt(proxyPort))));
             proxyExist = true;
-        }
+            PROXY_CLIENT = builder.build();
-        PROXY_CLIENT = builder.connectTimeout(10, TimeUnit.SECONDS)
+        } else {
-                .readTimeout(60, TimeUnit.SECONDS)
+            PROXY_CLIENT = CLIENT;
-                .build();
-    }
-
-    /***
-     * 下载文件
-     * @author fanxb
-     * @param url 下载链接
-     * @param proxy 是否使用代理
-     * @return java.io.InputStream
-     * @date 2021/3/12
-     **/
-    public static byte[] download(String url, boolean proxy) {
-        try (Response res = (proxy ? PROXY_CLIENT : CLIENT).newCall(new Request.Builder().url(url).build()).execute()) {
-            assert res.body() != null;
-            if (checkIsOk(res.code())) {
-                return res.body().byteStream().readAllBytes();
-            } else {
-                throw new CustomException("下载出现问题:" + res.body().string());
-            }
-        } catch (Exception e) {
-            throw new CustomException(e);
         }
     }
 
@@ -270,6 +259,8 @@ public class HttpUtil {
         }
         return ipAddress;
     }
+
+
 }
 
 

bookMarkService/files/public/icon/1.1614853232994.html

@@ -1,17 +0,0 @@
-<!DOCTYPE NETSCAPE-Bookmark-file-1>
-<!-- This is an automatically generated file.
-     It will be read and overwritten.
-     DO NOT EDIT! -->
-<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=UTF-8">
-<TITLE>Bookmarks</TITLE>
-<H1>Bookmarks Menu</H1>
-
-<DL><p>
-    <DT><A HREF="http://0.0.0.1/" ADD_DATE="1614837460" LAST_MODIFIED="1614837465">1</A>
-    <DT><A HREF="http://0.0.0.2/" ADD_DATE="1614837471" LAST_MODIFIED="1614837474">2</A>
-    <DT><H3 ADD_DATE="1614837478" LAST_MODIFIED="1614837497">f1</H3>
-    <DL><p>
-        <DT><A HREF="http://asdf/" ADD_DATE="1614837485" LAST_MODIFIED="1614837493" TAGS="ww">f11</A>
-        <DT><A HREF="http://f12/" ADD_DATE="1614837497" LAST_MODIFIED="1614837502">f12</A>
-    </DL><p>
-</DL>

bookMarkService/files/public/icon/1.1614853330412.html

@@ -1,17 +0,0 @@
-<!DOCTYPE NETSCAPE-Bookmark-file-1>
-<!-- This is an automatically generated file.
-     It will be read and overwritten.
-     DO NOT EDIT! -->
-<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=UTF-8">
-<TITLE>Bookmarks</TITLE>
-<H1>Bookmarks Menu</H1>
-
-<DL><p>
-    <DT><A HREF="http://0.0.0.1/" ADD_DATE="1614837460" LAST_MODIFIED="1614837465">1</A>
-    <DT><A HREF="http://0.0.0.2/" ADD_DATE="1614837471" LAST_MODIFIED="1614837474">2</A>
-    <DT><H3 ADD_DATE="1614837478" LAST_MODIFIED="1614837497">f1</H3>
-    <DL><p>
-        <DT><A HREF="http://asdf/" ADD_DATE="1614837485" LAST_MODIFIED="1614837493" TAGS="ww">f11</A>
-        <DT><A HREF="http://f12/" ADD_DATE="1614837497" LAST_MODIFIED="1614837502">f12</A>
-    </DL><p>
-</DL>

bookMarkService/web/src/main/resources/db/migration/V21__新增域名path表.sql

@@ -0,0 +1,10 @@
+CREATE TABLE bookmark.host_icon (
+	id INT UNSIGNED auto_increment NOT NULL,
+	host varchar(300) NOT NULL COMMENT 'host',
+	iconPath varchar(330) NOT NULL,
+	CONSTRAINT host_icon_pk PRIMARY KEY (id)
+)
+ENGINE=InnoDB
+DEFAULT CHARSET=utf8mb4
+COLLATE=utf8mb4_0900_ai_ci;
+CREATE INDEX host_icon_host_IDX USING BTREE ON bookmark.host_icon (host(20));

bookmark_front/src/views/noHead/addBookmark/index.vue

@@ -1,5 +1,8 @@
 <template>
-  <div class="ssoAddBookmark">正在添加，请稍后！！！</div>
+  <div class="ssoAddBookmark">
+    正在添加，请稍后！！！
+    <button @click="closeIframe">关闭</button>
+  </div>
 </template>
 
 <script>

浏览器插件/bookmarkBrowserPlugin/static/js/content.js

@@ -46,23 +46,14 @@ async function addBookmark (data) {
     return;
   }
   //新增书签
-  try {
-    if (data.data.iconUrl) {
-      let icon = await axios.get(data.data.iconUrl, { responseType: 'arraybuffer' });
-      data.data.icon = `data:` + icon.headers['content-type'] + ';base64,' + window.btoa(String.fromCharCode(...new Uint8Array(icon.data)));
-    }
-  } catch (error) {
-    console.error(error);
-  }
   console.log("新增书签", data.data);
   bookmarkInfo = data.data;
-
   addBlockDiv = document.createElement("div");
   addBlockDiv.setAttribute("style", "position:fixed;width:100%;height:100vh;z-index:100000;left:0;top:0;background:rgba(211, 211, 205, 0.8)");
   document.getElementsByTagName("body")[0].appendChild(addBlockDiv);
   iframe = document.createElement("iframe");
   iframe.src = bookmarkHost + "/noHead/addBookmark?token=" + data.token;
-  iframe.setAttribute("style", "width:70%;min-height:60vh;margin-left:15%;margin-top:10vh;padding:0.3em;");
+  iframe.setAttribute("style", "width:70%;min-height:60vh;margin-left:15%;margin-top:10vh;padding:0;border:0;border-radius:10px");
   addBlockDiv.appendChild(iframe);
 }