修改消息队列 发送

新增挂挡后发送识别队列
es查询测试编写
ocr识别批量插入队列编写
集成es

README.md

@@ -51,7 +51,8 @@ ______
 > * 日志目录：`/data/logs`
 > 
 > #### RabbitMQ
-> * 
+> * 使用docker安装
+> *
 
 ______
 

pom.xml

@@ -30,6 +30,7 @@
         <aliyun.sdk.version>4.5.3</aliyun.sdk.version>
         <aliyun.oss.sdk.version>3.10.2</aliyun.oss.sdk.version>
         <poi.version>4.1.2</poi.version>
+        <bboss.elasticsearch.version>6.2.2</bboss.elasticsearch.version>
         <!-- 跳过打包测试 -->
         <skipTests>true</skipTests>
     </properties>
@@ -137,6 +138,17 @@
                 </exclusion>
             </exclusions>
         </dependency>
+        <!--        bboss-->
+        <dependency>
+            <groupId>com.bbossgroups.plugins</groupId>
+            <artifactId>bboss-elasticsearch-rest-jdbc</artifactId>
+            <version>${bboss.elasticsearch.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.bbossgroups.plugins</groupId>
+            <artifactId>bboss-elasticsearch-spring-boot-starter</artifactId>
+            <version>${bboss.elasticsearch.version}</version>
+        </dependency>
     </dependencies>
 
     <build>

src/main/java/com/gz/config/WebSocketConfig.java

@@ -11,7 +11,7 @@ import org.springframework.web.socket.server.standard.ServerEndpointExporter;
  * @date 2020/8/7 9:50
  */
 @Slf4j
-@Configuration
+//@Configuration
 public class WebSocketConfig {
 
     /**

src/main/java/com/gz/dto/archive/ArchiveDTO.java

@@ -113,4 +113,7 @@ public class ArchiveDTO extends BaseDTO {
 
     // 全文标识
     private String qwbs;
+
+    // 现地籍号
+    private String xdjh;
 }

src/main/java/com/gz/job/MountArchiveFileJob.java

@@ -1,9 +1,11 @@
 package com.gz.job;
 
+import cn.hutool.core.bean.BeanUtil;
 import cn.hutool.core.date.DateUtil;
 import cn.hutool.core.io.FileUtil;
 import cn.hutool.core.util.ObjectUtil;
 import cn.hutool.core.util.ReUtil;
+import com.alibaba.fastjson.JSON;
 import com.gz.config.FileUploadConfig;
 import com.gz.config.MountArchiveFileConfig;
 import com.gz.core.exception.BusinessException;
@@ -16,7 +18,12 @@ import com.gz.mapper.archive.ArchiveFileMapper;
 import com.gz.mapper.archive.ArchiveMapper;
 import com.gz.mapper.archive.SecondaryArchiveMapper;
 import com.gz.mapper.system.ArchivesTreeMapper;
+import com.gz.rvo.archive.ArchiveRVO;
+import com.gz.vo.rabbitmq.OcrEtlVO;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.data.redis.core.StringRedisTemplate;
 import org.springframework.stereotype.Component;
 
@@ -49,6 +56,18 @@ public class MountArchiveFileJob {
     @Resource
     private StringRedisTemplate stringRedisTemplate;
 
+    @Value("${upload-file.server-file-url-prefix}")
+    private String serverFileUrlPrefix;
+
+    @Value("${rabbitmq.data-exchange}")
+    private String dataExchangeName;
+
+    @Value("${rabbitmq.ocr.queue-binding-key}")
+    private String ocrBindingKey;
+
+    @Autowired
+    private RabbitTemplate rabbitTemplate;
+
     @PostConstruct
     public void handelPDFJob() {
         log.info("[归档任务]初始化PDF文件归档任务线程");
@@ -69,16 +88,19 @@ public class MountArchiveFileJob {
                             key = String.format(MountArchiveFileConfig.ARCHIVE_FILE_MOUNT_ERROR_FOMAT, pdf.getName());
                             // 判断是否在错误队列 错误队列跳过
                             if (!stringRedisTemplate.hasKey(key)) {
-                                ArchiveDTO archiveDTO = new ArchiveDTO();
+                                // 案卷dto
+                                ArchiveRVO archiveDTO = new ArchiveRVO();
+                                // 卷内目录dto
                                 SecondaryArchiveDTO secondaryArchiveDTO = new SecondaryArchiveDTO();
                                 // 专业档案卷内目录 档号正则
                                 if (ReUtil.isMatch(MountArchiveFileConfig.ZY_DH_RE, dh)) {
                                     secondaryArchiveDTO.setDh(dh);
                                     secondaryArchiveDTO = secondaryArchiveMapper.selectOne(secondaryArchiveDTO);
-                                    archiveDTO = archiveMapper.selectByPrimaryKey(secondaryArchiveDTO.getArchiveId());
+                                    archiveDTO = archiveMapper.selectByPk(secondaryArchiveDTO.getArchiveId());
                                 } else {
                                     archiveDTO.setDh(dh);
-                                    archiveDTO = archiveMapper.selectOne(archiveDTO);
+                                    archiveDTO = archiveMapper.selectByDh(archiveDTO.getDh());
+
                                 }
                                 if (archiveDTO == null) {
                                     throw new BusinessException(CustomExceptionEnum.ARCHIVE_DATA_NOT_EXISTS);
@@ -86,7 +108,7 @@ public class MountArchiveFileJob {
                                 ArchivesTreeDTO parentMl = archivesTreeMapper.selectParentByCode(archiveDTO.getMl());
                                 String mlDirectory = (parentMl == null) ? archiveDTO.getMl() : parentMl.getCode() + "/" + archiveDTO.getMl();
                                 String finallyPath = "/" + archiveDTO.getQzh() + "/" + archiveDTO.getGdnd() + "/" + mlDirectory + "/" + archiveDTO.getBgqx() + "/" + archiveDTO.getDh() + "/pdf/";
-                                FileUtil.copy(pdf, new File(FileUploadConfig.ARCHIVE_FILE_ROOT_DIRECTORY + finallyPath + name), false);
+                                FileUtil.copy(pdf, new File(FileUploadConfig.ARCHIVE_FILE_ROOT_DIRECTORY + finallyPath + name), true);
                                 log.debug("复制文件:[{}]成功", name);
                                 // 删除扫描成功文件
                                 if (!FileUtil.del(pdf)) {
@@ -102,8 +124,23 @@ public class MountArchiveFileJob {
                                 archiveFileDTO.setFileSize(FileUtil.size(pdf));
                                 archiveFileDTO.setFilePath(finallyPath);
                                 archiveFileDTO.setFileType(0);
-                                archiveFileMapper.insertSelective(archiveFileDTO);
+                                if (archiveFileMapper.selectOne(archiveFileDTO) == null) {
+                                    archiveFileMapper.insertSelective(archiveFileDTO);
+                                } else {
+                                    log.info("有文件记录跳过");
+                                }
                                 // TODO: 2021/3/17 发送rabbitmq 队列消息
+                                if (ObjectUtil.isNotNull(archiveDTO)) {
+                                    OcrEtlVO ocrBean = BeanUtil.copyProperties(archiveDTO, OcrEtlVO.class);
+                                    ocrBean.setFileUrl(serverFileUrlPrefix + archiveFileDTO.getFilePath() + "/" + archiveFileDTO.getFileName());
+                                    if (ObjectUtil.isNotNull(secondaryArchiveDTO)) {
+                                        ocrBean.setSecondaryArchiveId(secondaryArchiveDTO.getId());
+                                    } else {
+                                        ocrBean.setSecondaryArchiveId(null);
+                                    }
+                                    log.info("【ocr识别】发起识别队列，参数[{}]", JSON.toJSONString(ocrBean));
+                                    rabbitTemplate.convertAndSend(dataExchangeName, ocrBindingKey, JSON.toJSONString(ocrBean));
+                                }
                             }
                         } catch (Exception e) {
                             log.error("挂载出现错误:", e);

src/main/java/com/gz/mapper/archive/ArchiveMapper.java

@@ -22,6 +22,10 @@ public interface ArchiveMapper extends Mapper<ArchiveDTO> {
 
     List<ArchiveRVO> againSearchList(AgainSearchVO vo);
 
+    ArchiveRVO selectByPk(Integer id);
+
+    ArchiveRVO selectByDh(String dh);
+
     /**
      * @description 查询当前件号
      * @author LiuChangLan

src/main/java/com/gz/scheduled/ScanArchiveFileScheduled.java

@@ -5,6 +5,7 @@ import cn.hutool.core.util.StrUtil;
 import com.gz.config.MountArchiveFileConfig;
 import com.gz.core.exception.BusinessException;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Profile;
 import org.springframework.data.redis.core.StringRedisTemplate;
 import org.springframework.scheduling.annotation.Scheduled;

src/main/java/com/gz/service/archive/impl/ArchiveFileServiceImpl.java

@@ -16,6 +16,7 @@ import com.gz.mapper.archive.ArchiveFileMapper;
 import com.gz.mapper.archive.ArchiveMapper;
 import com.gz.mapper.system.MenuRoleMapper;
 import com.gz.rvo.archive.ArchiveFileRVO;
+import com.gz.rvo.archive.ArchiveRVO;
 import com.gz.service.archive.ArchiveFileService;
 import com.gz.utils.JwtUtils;
 import com.gz.vo.archive.SearchArchiveFileVO;
@@ -134,7 +135,7 @@ public class ArchiveFileServiceImpl implements ArchiveFileService {
 
 
         if (fileType == 0) {
-            ArchiveDTO d = archiveMapper.selectByPrimaryKey(archiveFileDTO.getArchiveId());
+            ArchiveRVO d = archiveMapper.selectByPk(archiveFileDTO.getArchiveId());
             OcrEtlVO ocrEtlVO = BeanUtil.copyProperties(d, OcrEtlVO.class);
             ocrEtlVO.setSecondaryArchiveId(secondaryArchiveId);
             ocrEtlVO.setFileUrl(serverFileUrlPrefix + archiveFileDTO.getFilePath() + "/" + archiveFileDTO.getFileName());

src/main/java/com/gz/service/archive/impl/ArchiveServiceImpl.java

@@ -4,6 +4,7 @@ import cn.hutool.core.bean.BeanUtil;
 import cn.hutool.core.date.DateField;
 import cn.hutool.core.date.DateTime;
 import cn.hutool.core.date.DateUtil;
+import cn.hutool.core.exceptions.ExceptionUtil;
 import cn.hutool.core.util.StrUtil;
 import cn.hutool.poi.excel.ExcelReader;
 import cn.hutool.poi.excel.ExcelUtil;
@@ -336,6 +337,8 @@ public class ArchiveServiceImpl implements ArchiveService {
                 dto.setZtc(ExcelUtils.getExcelCellStringValue(row, 26));
                 dto.setQwbs(ExcelUtils.getExcelCellStringValue(row, 27));
                 dto.setMlh(ExcelUtils.getExcelCellStringValue(row, 28));
+                dto.setYs(ExcelUtils.getExcelCellIntegerValue(row, 31));
+                dto.setXdjh(ExcelUtils.getExcelCellStringValue(row, 32));
 //                dto.setMj("NABU");
                 result.add(dto);
             } catch (Exception e) {
@@ -374,7 +377,10 @@ public class ArchiveServiceImpl implements ArchiveService {
                 // 日期
                 if (StrUtil.isEmpty(ExcelUtils.getExcelCellStringValue(row, 5))) {
                     dto.setRq("");
-                } else {
+                } else if (ExcelUtils.getExcelCellStringValue(row, 5).length() == 6) {
+                    DateTime rq = DateUtil.parse(ExcelUtils.getExcelCellStringValue(row, 5), "yyyyMM");
+                    dto.setRq(DateUtil.formatDate(rq));
+                } else if (ExcelUtils.getExcelCellStringValue(row, 5).length() == 8) {
                     DateTime rq = DateUtil.parse(ExcelUtils.getExcelCellStringValue(row, 5), "yyyyMMdd");
                     dto.setRq(DateUtil.formatDate(rq));
                 }

src/main/java/com/gz/vo/rabbitmq/OcrEtlVO.java

@@ -1,6 +1,7 @@
 package com.gz.vo.rabbitmq;
 
 import com.gz.dto.archive.ArchiveDTO;
+import com.gz.rvo.archive.ArchiveRVO;
 import lombok.Data;
 
 /**
@@ -8,7 +9,7 @@ import lombok.Data;
  * @date 2021/3/9 9:45
  */
 @Data
-public class OcrEtlVO extends ArchiveDTO {
+public class OcrEtlVO extends ArchiveRVO {
     // 文件路径
     private String fileUrl;
 

src/main/resources/application-prod.yml

@@ -78,7 +78,7 @@ spring:
     # 连接超时时间（毫秒）
     timeout: 30000
   rabbitmq:
-    host: 127.0.0.1
+    host: 66.1.21.158
     port: 5672
     username: admin
     password: MjmaAH4lg5o84XXj
@@ -90,6 +90,33 @@ spring:
           enabled: true # 是否开启消费者重试（为false时关闭消费者重试，这时消费端代码异常会一直重复收到消息）
           max-attempts: 5 # 重试次数
           initial-interval: 5000
+  elasticsearch:
+    bboss:
+      elasticsearch:
+        rest:
+          hostNames: 66.1.21.159:9200
+        dateFormat: yyyy.MM.dd
+        timeZone: Asia/Shanghai
+        showTemplate: true
+        discoverHost: false
+      dslfile:
+        refreshInterval: -1
+      http:
+        timeoutConnection: 5000
+        timeoutSocket: 5000
+        connectionRequestTimeout: 5000
+        retryTime: 1
+        maxLineLength: -1
+        maxHeaderCount: 200
+        maxTotal: 400
+        defaultMaxPerRoute: 200
+        soReuseAddress: false
+        soKeepAlive: false
+        timeToLive: 3600000
+        keepAlive: 3600000
+        keystore:
+        keyPassword:
+        hostnameVerifier:
 upload-file:
   server-file-url-prefix: http://66.1.21.158:${server.port}
   file-root-directory: /data/file

src/main/resources/application.yml

@@ -1,6 +1,6 @@
 spring:
   profiles:
-    active: dev
+    active: prod
   servlet:
     multipart:
       max-file-size: 5120MB

src/main/resources/elasticsearch/archive.xml

@@ -0,0 +1,13 @@
+<properties>
+    <property name="query">
+        <![CDATA[
+         {
+              "query": {
+                "match_all": {}
+              },
+              "from": #[pageIndex],
+              "size": #[pageSize]
+          }
+        ]]>
+    </property>
+</properties>

src/main/resources/mapper/archiveMapper.xml

@@ -198,4 +198,33 @@
           and atr.deleted = 0
           and a.id = #{id}
     </select>
+
+    <select id="selectByPk" resultType="com.gz.rvo.archive.ArchiveRVO" parameterType="int">
+        SELECT
+            d.dict_name mj_text,
+            d1.dict_name bgqx_text,
+            d2.title ml_Text,
+            a.*
+        FROM `tab_archives` a
+         LEFT JOIN tab_dict d ON a.mj = d.dict_code
+         LEFT JOIN tab_dict d1 ON a.bgqx = d1.dict_code
+         LEFT JOIN tab_archives_tree d2 ON a.ml = d2.code
+        where a.deleted = 0
+        and a.id = #{id}
+    </select>
+
+
+    <select id="selectByDh"  resultType="com.gz.rvo.archive.ArchiveRVO" parameterType="string">
+        SELECT
+            d.dict_name mj_text,
+            d1.dict_name bgqx_text,
+            d2.title ml_Text,
+            a.*
+        FROM `tab_archives` a
+                 LEFT JOIN tab_dict d ON a.mj = d.dict_code
+                 LEFT JOIN tab_dict d1 ON a.bgqx = d1.dict_code
+                 LEFT JOIN tab_archives_tree d2 ON a.ml = d2.code
+        where a.deleted = 0
+          and a.dh = #{dh}
+    </select>
 </mapper>

src/test/java/com/gz/ESTest.java

@@ -0,0 +1,34 @@
+package com.gz;
+
+import com.alibaba.fastjson.JSON;
+import org.frameworkset.elasticsearch.ElasticSearchHelper;
+import org.frameworkset.elasticsearch.boot.BBossESStarter;
+import org.frameworkset.elasticsearch.client.ClientInterface;
+import org.frameworkset.elasticsearch.entity.ESDatas;
+import org.junit.jupiter.api.Test;
+import org.springframework.boot.test.context.SpringBootTest;
+
+import javax.annotation.Resource;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author LiuchangLan
+ * @date 2021/4/26 19:49
+ */
+@SpringBootTest
+public class ESTest {
+
+    @Resource
+    private BBossESStarter bbossESStarter;
+
+    @Test
+    void test(){
+        ClientInterface restClient = ElasticSearchHelper.getConfigRestClientUtil("elasticsearch/archive.xml");
+        Map<String,Object> params = new HashMap<>();
+        params.put("pageIndex",1);
+        params.put("pageSize",10);
+        ESDatas<Map> mapESDatas = restClient.searchList("/hp_es/_doc/_search", "query", params, Map.class);
+        System.out.println(JSON.toJSONString(mapESDatas));
+    }
+}

src/test/java/com/gz/OcrTest.java

@@ -0,0 +1,80 @@
+package com.gz;
+
+import cn.hutool.core.bean.BeanUtil;
+import cn.hutool.core.util.StrUtil;
+import com.alibaba.fastjson.JSON;
+import com.gz.dto.archive.ArchiveDTO;
+import com.gz.dto.archive.SecondaryArchiveDTO;
+import com.gz.dto.system.ArchiveFileDTO;
+import com.gz.dto.system.ArchivesTreeDTO;
+import com.gz.mapper.archive.ArchiveFileMapper;
+import com.gz.mapper.archive.ArchiveMapper;
+import com.gz.mapper.archive.SecondaryArchiveMapper;
+import com.gz.rvo.archive.ArchiveRVO;
+import com.gz.vo.rabbitmq.OcrEtlVO;
+import lombok.extern.slf4j.Slf4j;
+import org.junit.jupiter.api.Test;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.data.redis.core.StringRedisTemplate;
+import tk.mybatis.mapper.entity.Example;
+
+import javax.annotation.Resource;
+import java.util.List;
+
+/**
+ * @author LiuchangLan
+ * @date 2021/4/26 18:50
+ */
+@Slf4j
+@SpringBootTest
+public class OcrTest {
+
+    @Resource
+    private ArchiveMapper archiveMapper;
+
+    @Resource
+    private SecondaryArchiveMapper secondaryArchiveMapper;
+
+    @Resource
+    private StringRedisTemplate stringRedisTemplate;
+
+    @Value("${upload-file.server-file-url-prefix}")
+    private String serverFileUrlPrefix;
+
+    @Value("${rabbitmq.data-exchange}")
+    private String dataExchangeName;
+
+    @Autowired
+    private RabbitTemplate rabbitTemplate;
+
+    @Resource
+    private ArchiveFileMapper archiveFileMapper;
+
+    @Value("${rabbitmq.ocr.queue-binding-key}")
+    private String ocrBindingKey;
+
+
+    @Test
+    public void test(){
+//        ArchiveRVO archiveRVO = archiveMapper.selectByDh("5179-WS2019-001-0000-0005");
+//        System.out.println(1);
+//        Example example = new Example(ArchiveFileDTO.class);
+//        example.setOrderByClause("id ASC");
+//        List<ArchiveFileDTO> archiveFileDTOS = archiveFileMapper.selectByExample(example);
+//        Integer i = Integer.valueOf(stringRedisTemplate.opsForValue().get("sdfsdf"));
+//        while (i < archiveFileDTOS.size()){
+//            ArchiveFileDTO archiveFileDTO = archiveFileDTOS.get(i);
+//            ArchiveRVO archiveRVO = archiveMapper.selectByPk(archiveFileDTO.getArchiveId());
+//            OcrEtlVO ocrBean = BeanUtil.copyProperties(archiveRVO, OcrEtlVO.class);
+//            ocrBean.setSecondaryArchiveId(archiveFileDTO.getSecondaryArchiveId());
+//            ocrBean.setFileUrl(serverFileUrlPrefix + archiveFileDTO.getFilePath() + "/" + archiveFileDTO.getFileName());
+//            log.info("【ocr识别】发起识别队列，参数[{}]", JSON.toJSONString(ocrBean));
+//            rabbitTemplate.convertAndSend(dataExchangeName, ocrBindingKey, JSON.toJSONString(ocrBean));
+//            i ++;
+//            stringRedisTemplate.opsForValue().set("sdfsdf",i.toString());
+//        }
+    }
+}