diff --git a/qieziBackend/src/main/java/com/fanxb/backend/constants/RedisConstant.java b/qieziBackend/src/main/java/com/fanxb/backend/constants/RedisConstant.java index d9cbc9b..5f47d06 100644 --- a/qieziBackend/src/main/java/com/fanxb/backend/constants/RedisConstant.java +++ b/qieziBackend/src/main/java/com/fanxb/backend/constants/RedisConstant.java @@ -22,4 +22,9 @@ public class RedisConstant { */ public static final String HOST_UV_PRE = "host_uv_pre_"; public static final String PAGE_UV_PRE = "page_uv_pre_"; + /** + * 日统计数据id key前缀 + */ + public static final String DAY_HOST_ID_PRE = "day_host_id_pre_"; + public static final String DAY_DETAIL_PAGE_ID_PRE = "day_detail_page_id_pre_"; } diff --git a/qieziBackend/src/main/java/com/fanxb/backend/controller/ApplicationController.java b/qieziBackend/src/main/java/com/fanxb/backend/controller/ApplicationController.java index a41bff3..fdf27e5 100644 --- a/qieziBackend/src/main/java/com/fanxb/backend/controller/ApplicationController.java +++ b/qieziBackend/src/main/java/com/fanxb/backend/controller/ApplicationController.java @@ -46,7 +46,7 @@ public class ApplicationController { @GetMapping("/visit") @ResponseBody public void visit(HttpServletRequest request, HttpServletResponse response, @NotBlank(message = "回调函数不能为空") String callBack - , @NotBlank(message = "key不能为空") String key, @NotBlank(message = "path不能为空") String path, boolean notAdd) throws IOException { + , @NotBlank(message = "key不能为空") String key, @NotBlank(message = "path不能为空") String path, @RequestParam(defaultValue = "false") boolean notAdd) throws IOException { applicationService.visit(request, response, callBack, key, path, notAdd); } } diff --git a/qieziBackend/src/main/java/com/fanxb/backend/dao/DetailPageDao.java b/qieziBackend/src/main/java/com/fanxb/backend/dao/DetailPageDao.java index 66f3cb0..1a34251 100644 --- a/qieziBackend/src/main/java/com/fanxb/backend/dao/DetailPageDao.java +++ b/qieziBackend/src/main/java/com/fanxb/backend/dao/DetailPageDao.java @@ -20,6 +20,7 @@ public interface DetailPageDao { * date 2022/2/15 16:39 */ @Insert("insert into detail_page(hostId,path,pv,uv) value(#{hostId},#{path},#{pv},#{uv})") + @Options(useGeneratedKeys = true, keyProperty = "id", keyColumn = "id") void insertOne(DetailPagePo detailPagePo); /** diff --git a/qieziBackend/src/main/java/com/fanxb/backend/dao/DetailPageDayDao.java b/qieziBackend/src/main/java/com/fanxb/backend/dao/DetailPageDayDao.java new file mode 100644 index 0000000..a6bcc42 --- /dev/null +++ b/qieziBackend/src/main/java/com/fanxb/backend/dao/DetailPageDayDao.java @@ -0,0 +1,42 @@ +package com.fanxb.backend.dao; + +import com.fanxb.backend.entity.po.DetailPageDayPo; +import com.fanxb.backend.entity.po.HostDayPo; +import org.apache.ibatis.annotations.*; + +/** + * @author fanxb + */ +@Mapper +public interface DetailPageDayDao { + /** + * 获取id + * + * @param detailPageId id + * @param dayNum day + * @author fanxb + */ + @Select("select id from detail_page_day where detailPageId=#{detailPageId} and dayNum=#{dayNum}") + Integer getId(@Param("detailPageId") int detailPageId, @Param("dayNum") int dayNum); + + /** + * 插入一条数据 + * + * @param po po + * @author fanxb + */ + @Insert("insert into detail_page_day(detailPageId,dayNum,pv,uv) value(#{detailPageId},#{dayNum},0,0)") + @Options(useGeneratedKeys = true, keyProperty = "id", keyColumn = "id") + void insert(DetailPageDayPo po); + + /** + * 更新pv,uv + * + * @param id id + * @param pvIncrement pv增量 + * @param uvIncrement uv增量 + * @author fanxb + */ + @Update("update detail_page_day set uv=uv+#{uvIncrement},pv=pv+#{pvIncrement} where id=#{id}") + void updatePvUv(@Param("id") int id, @Param("pvIncrement") int pvIncrement, @Param("uvIncrement") int uvIncrement); +} diff --git a/qieziBackend/src/main/java/com/fanxb/backend/dao/HostDayDao.java b/qieziBackend/src/main/java/com/fanxb/backend/dao/HostDayDao.java new file mode 100644 index 0000000..d6b257d --- /dev/null +++ b/qieziBackend/src/main/java/com/fanxb/backend/dao/HostDayDao.java @@ -0,0 +1,41 @@ +package com.fanxb.backend.dao; + +import com.fanxb.backend.entity.po.HostDayPo; +import org.apache.ibatis.annotations.*; + +/** + * @author fanxb + */ +@Mapper +public interface HostDayDao { + /** + * 获取id + * + * @param hostId hostId + * @param dayNum day + * @author fanxb + */ + @Select("select id from host_day where hostId=#{hostId} and dayNum=#{dayNum}") + Integer getId(@Param("hostId") int hostId, @Param("dayNum") int dayNum); + + /** + * 插入一条数据 + * + * @param hostDayPo po + * @author fanxb + */ + @Insert("insert into host_day(hostId,dayNum,pv,uv) value(#{hostId},#{dayNum},0,0)") + @Options(useGeneratedKeys = true, keyProperty = "id", keyColumn = "id") + void insert(HostDayPo hostDayPo); + + /** + * 更新pv,uv + * + * @param id id + * @param pvIncrement pv增量 + * @param uvIncrement uv增量 + * @author fanxb + */ + @Update("update host_day set uv=uv+#{uvIncrement},pv=pv+#{pvIncrement} where id=#{id}") + void updatePvUv(@Param("id") int id, @Param("pvIncrement") int pvIncrement, @Param("uvIncrement") int uvIncrement); +} diff --git a/qieziBackend/src/main/java/com/fanxb/backend/entity/po/DetailPageDayPo.java b/qieziBackend/src/main/java/com/fanxb/backend/entity/po/DetailPageDayPo.java new file mode 100644 index 0000000..b9191f8 --- /dev/null +++ b/qieziBackend/src/main/java/com/fanxb/backend/entity/po/DetailPageDayPo.java @@ -0,0 +1,26 @@ +package com.fanxb.backend.entity.po; + + +import lombok.Data; +import lombok.experimental.Accessors; + +@Data +@Accessors(chain = true) +public class DetailPageDayPo { + + /** + * id + */ + private int id; + /** + * hostId + */ + private int detailPageId; + /** + * 日期20200202 + */ + private int dayNum; + private long pv; + private long uv; + +} diff --git a/qieziBackend/src/main/java/com/fanxb/backend/entity/po/HostDayPo.java b/qieziBackend/src/main/java/com/fanxb/backend/entity/po/HostDayPo.java index e16f25a..d0da58d 100644 --- a/qieziBackend/src/main/java/com/fanxb/backend/entity/po/HostDayPo.java +++ b/qieziBackend/src/main/java/com/fanxb/backend/entity/po/HostDayPo.java @@ -2,8 +2,10 @@ package com.fanxb.backend.entity.po; import lombok.Data; +import lombok.experimental.Accessors; @Data +@Accessors(chain = true) public class HostDayPo { /** @@ -17,7 +19,7 @@ public class HostDayPo { /** * 日期20200202 */ - private int dateNum; + private int dayNum; private long pv; private long uv; diff --git a/qieziBackend/src/main/java/com/fanxb/backend/service/impl/ApplicationServiceImpl.java b/qieziBackend/src/main/java/com/fanxb/backend/service/impl/ApplicationServiceImpl.java index 01b57cc..6b46e57 100644 --- a/qieziBackend/src/main/java/com/fanxb/backend/service/impl/ApplicationServiceImpl.java +++ b/qieziBackend/src/main/java/com/fanxb/backend/service/impl/ApplicationServiceImpl.java @@ -1,23 +1,25 @@ package com.fanxb.backend.service.impl; import cn.hutool.core.util.IdUtil; -import cn.hutool.core.util.StrUtil; import cn.hutool.http.Header; import com.alibaba.fastjson.JSON; -import com.fanxb.backend.constants.RedisConstant; import com.fanxb.backend.constants.CommonConstant; +import com.fanxb.backend.constants.RedisConstant; import com.fanxb.backend.dao.DetailPageDao; +import com.fanxb.backend.dao.DetailPageDayDao; import com.fanxb.backend.dao.HostDao; +import com.fanxb.backend.dao.HostDayDao; import com.fanxb.backend.entity.dto.ApplicationSignDto; import com.fanxb.backend.entity.exception.CustomBaseException; +import com.fanxb.backend.entity.po.DetailPageDayPo; import com.fanxb.backend.entity.po.DetailPagePo; +import com.fanxb.backend.entity.po.HostDayPo; import com.fanxb.backend.entity.po.HostPo; import com.fanxb.backend.entity.vo.ApplicationSignVo; import com.fanxb.backend.entity.vo.UvPvVo; import com.fanxb.backend.service.ApplicationService; import com.fanxb.backend.util.NetUtil; import com.fanxb.backend.util.ThreadPoolUtil; -import jdk.jfr.ContentType; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.stereotype.Service; @@ -25,11 +27,10 @@ import org.springframework.stereotype.Service; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.IOException; -import java.net.URL; import java.nio.charset.StandardCharsets; import java.time.LocalDate; import java.time.ZoneId; -import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; import java.util.concurrent.TimeUnit; /** @@ -40,15 +41,20 @@ import java.util.concurrent.TimeUnit; */ @Service public class ApplicationServiceImpl implements ApplicationService { + private final StringRedisTemplate stringRedisTemplate; private final HostDao hostDao; private final DetailPageDao detailPageDao; + private final HostDayDao hostDayDao; + private final DetailPageDayDao detailPageDayDao; @Autowired - public ApplicationServiceImpl(StringRedisTemplate stringRedisTemplate, HostDao hostDao, DetailPageDao detailPageDao) { + public ApplicationServiceImpl(StringRedisTemplate stringRedisTemplate, HostDao hostDao, DetailPageDao detailPageDao, HostDayDao hostDayDao, DetailPageDayDao detailPageDayDao) { this.stringRedisTemplate = stringRedisTemplate; this.hostDao = hostDao; this.detailPageDao = detailPageDao; + this.hostDayDao = hostDayDao; + this.detailPageDayDao = detailPageDayDao; } @Override @@ -94,10 +100,10 @@ public class ApplicationServiceImpl implements ApplicationService { * @param hostPo hostPo * @param detailPagePo detailPagePo * @author fanxb - * date 2022/2/16 15:40 */ private void updateData(String ip, HostPo hostPo, DetailPagePo detailPagePo) { String hostKey = RedisConstant.HOST_UV_PRE + hostPo.getId() + "_" + ip; + //记录ip是否在当天访问过host String hostVal = stringRedisTemplate.opsForValue().get(hostKey); hostPo.setPv(hostPo.getPv() + 1); long tomorrowZero = LocalDate.now().plusDays(1).atStartOfDay().atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); @@ -106,6 +112,7 @@ public class ApplicationServiceImpl implements ApplicationService { hostPo.setUv(hostPo.getUv() + 1); } String pageKey = RedisConstant.PAGE_UV_PRE + hostPo.getId() + ip + detailPagePo.getPath(); + //记录ip是否在当天访问过页面 String pageVal = stringRedisTemplate.opsForValue().get(pageKey); detailPagePo.setPv(detailPagePo.getPv() + 1); if (pageVal == null) { @@ -121,16 +128,54 @@ public class ApplicationServiceImpl implements ApplicationService { detailPageDao.updateUvPv(detailPagePo.getId(), pageVal == null ? 1 : 0); } //更新站点的日pv,uv + int dayNum = Integer.parseInt(LocalDate.now().format(DateTimeFormatter.ISO_LOCAL_DATE).replaceAll("-", "")); + hostDayDao.updatePvUv(getDayTableId(hostPo.getId(), dayNum, RedisConstant.DAY_HOST_ID_PRE), 1, hostVal == null ? 1 : 0); + detailPageDayDao.updatePvUv(getDayTableId(detailPagePo.getId(), dayNum, RedisConstant.DAY_DETAIL_PAGE_ID_PRE), 1, pageVal == null ? 1 : 0); }); } + /** + * 获取表id，数据行不存在时新增行 + * + * @param id id + * @param day day(20220202) + * @param type 类型 + * @author fanxb + */ + private int getDayTableId(int id, int day, String type) { + String key = type + "_" + id + "_" + day; + String val = stringRedisTemplate.opsForValue().get(key); + if (val != null) { + return Integer.parseInt(val); + } + //redis中没有,从数据库中查找 + Integer dbId = RedisConstant.DAY_HOST_ID_PRE.equals(type) ? hostDayDao.getId(id, day) : detailPageDayDao.getId(id, day); + if (dbId == null) { + //TODO 此次应该加锁，避免并发情况下重复创建报错 + dbId = RedisConstant.DAY_HOST_ID_PRE.equals(type) ? hostDayDao.getId(id, day) : detailPageDayDao.getId(id, day); + if (dbId == null) { + //数据库中也没有，需要先初始化数据库中数据 + if (RedisConstant.DAY_HOST_ID_PRE.equals(type)) { + HostDayPo hostDayPo = new HostDayPo().setHostId(id).setDayNum(day); + hostDayDao.insert(hostDayPo); + dbId = hostDayPo.getId(); + } else { + DetailPageDayPo detailPageDayPo = new DetailPageDayPo().setDetailPageId(id).setDayNum(day); + detailPageDayDao.insert(detailPageDayPo); + dbId = detailPageDayPo.getId(); + } + } + } + stringRedisTemplate.opsForValue().set(key, dbId.toString()); + return dbId; + } + /** * 获取hostID * * @param key key * @return {@link int} * @author fanxb - * date 2022/2/16 15:37 */ private int getHostId(String key) { String id = stringRedisTemplate.opsForValue().get(key); @@ -141,7 +186,7 @@ public class ApplicationServiceImpl implements ApplicationService { if (hostId == null) { throw new CustomBaseException("key无效:" + key); } - stringRedisTemplate.opsForValue().set(key, hostId.toString()); + stringRedisTemplate.opsForValue().set(key, hostId.toString(), 2, TimeUnit.DAYS); return hostId; } diff --git a/qieziBackend/src/main/java/com/fanxb/backend/util/RedisLockUtil.java b/qieziBackend/src/main/java/com/fanxb/backend/util/RedisLockUtil.java new file mode 100644 index 0000000..a12da6e --- /dev/null +++ b/qieziBackend/src/main/java/com/fanxb/backend/util/RedisLockUtil.java @@ -0,0 +1,14 @@ +package com.fanxb.backend.util; + +import org.springframework.stereotype.Component; + +/** + * redis分布式锁操作工具类 + * + * @author fanxb + */ +@Component +public class RedisLockUtil { + + +} diff --git a/qieziBackend/src/main/resources/db/migration/V2__修改日pv,uv表.sql b/qieziBackend/src/main/resources/db/migration/V2__修改日pv,uv表.sql index a86f318..d67f236 100644 --- a/qieziBackend/src/main/resources/db/migration/V2__修改日pv,uv表.sql +++ b/qieziBackend/src/main/resources/db/migration/V2__修改日pv,uv表.sql @@ -1,25 +1,29 @@ drop table host_day; CREATE TABLE qiezi.host_day ( + id INT auto_increment NOT NULL, hostId int NOT NULL COMMENT 'host表主键', dayNum INT NOT NULL COMMENT '日期：20200202', uv INT NOT NULL, pv int NOT NULL, - CONSTRAINT host_day_pk PRIMARY KEY (hostId,dayNum) + CONSTRAINT host_day_pk PRIMARY KEY (id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='整站，日 uv/pv记录'; +CREATE UNIQUE INDEX host_day_key_IDX USING BTREE ON qiezi.host_day (hostId,dayNum); CREATE TABLE qiezi.detail_page_day ( + id INT auto_increment NOT NULL, detailPageId int NOT NULL COMMENT 'detail_page主键', dayNum INT NOT NULL COMMENT '日期：20200202', uv INT NOT NULL, pv int NOT NULL, - CONSTRAINT detail_page_day_pk PRIMARY KEY (detailPageId,dayNum) + CONSTRAINT detail_page_day_pk PRIMARY KEY (id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='具体页面，日 uv/pv记录'; +CREATE UNIQUE INDEX detail_page_day_key_IDX USING BTREE ON qiezi.detail_page_day(detailPageId,dayNum);