dev

common/pom.xml

@@ -1,5 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
     <parent>
         <groupId>work.xuye</groupId>
@@ -120,6 +121,10 @@
             <artifactId>sq-sentry</artifactId>
             <version>1.1.8</version>
         </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-validation</artifactId>
+        </dependency>
 
     </dependencies>
 

common/src/main/java/work/xuye/common/service/MessageService.java

@@ -58,7 +58,7 @@ public class MessageService {
                     .send();
             messageMD5Store.save(result.toString());
         } else {
-            log.debug("短期内已经发送过相同的异常信息，不再发送: [{}]", result);
+            log.info("短期内已经发送过相同的异常信息，不再发送: [{}]", result);
         }
     }
 }

helper/src/main/java/work/xuye/helper/checker/DataDiffChecker.java

@@ -0,0 +1,180 @@
+package work.xuye.helper.checker;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.stereotype.Service;
+import org.springframework.util.ObjectUtils;
+import work.xuye.common.db.service.TaskManager;
+import work.xuye.helper.properties.HelperProperties;
+import work.xuye.helper.service.JdbcTemplateManager;
+
+import java.util.*;
+
+/**
+ * @author xuye
+ * @since 2023/7/13 14:57
+ **/
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class DataDiffChecker {
+
+    private final TaskManager taskManager;
+    private final JdbcTemplateManager jdbcTemplateManager;
+    private final HelperProperties helperProperties;
+
+
+    public Map<String, List<Object>> checkDifferentKeys(Map<String, Object> map1, Map<String, Object> map2) {
+        Map<String, List<Object>> map = new HashMap<>();
+        for (String key : map1.keySet()) {
+            if (map2.containsKey(key)) {
+                Object value1 = map1.get(key);
+                Object value2 = map2.get(key);
+
+                if (!ObjectUtils.nullSafeEquals(value1, value2)) {
+                    if (!helperProperties.getEqualsIgnoreField().contains(key)) {
+                        map.put(key, List.of(value1, value2));
+                        log.debug("@@@ key: {}, value1: {}, value2: {}", key, value1, value2);
+                    }
+                }
+            } else {
+                log.debug("@@@ key: {}, value1: {}, value2: {}", key, map1.get(key), null);
+                ArrayList<Object> list = new ArrayList<>();
+                list.add(map1.get(key));
+                list.add(null);
+                map.put(key, list);
+            }
+        }
+        for (String key : map2.keySet()) {
+            if (!map1.containsKey(key)) {
+                log.debug("@@@ key: {}, value1: {}, value2: {}", key, null, map2.get(key));
+                ArrayList<Object> list = new ArrayList<>();
+                list.add(null);
+                list.add(map2.get(key));
+                map.put(key, list);
+            }
+        }
+        return map;
+    }
+
+
+    public void check(List<Integer> ids, Integer intervalDays) {
+        List<Integer> newsIds = helperProperties.getTaskTypeIdMap().get(NewsTypeEnum.NEWS);
+        List<Integer> videoIds = helperProperties.getTaskTypeIdMap().get(NewsTypeEnum.VIDEO);
+        if (ObjectUtils.isEmpty(ids)) {
+            handleNews(newsIds, intervalDays);
+//            handleVideo(videoIds);
+        } else {
+            for (Integer id : ids) {
+                if (newsIds.contains(id)) {
+                    handleNews(List.of(id), intervalDays);
+                } else if (videoIds.contains(id)) {
+//                    handleVideo(List.of(id));
+                }
+            }
+        }
+    }
+
+    public void handleNews(List<Integer> ids, Integer intervalDays) {
+
+        String source = helperProperties.getSourceDbName();
+        String target = helperProperties.getTargetDbName();
+
+        String sql = "select url from all_news where spider = ? and createDate > CURDATE() - INTERVAL ? DAY order by createDate desc";
+        for (Integer id : ids) {
+            String taskName = taskManager.getTaskInfoByTaskId(id).getTask().getName();
+            log.info("-------------------------{}-------------------------", taskName);
+            List<String> sourceUrl = jdbcTemplateManager.getTemplate(source).queryForList(sql, String.class, taskName, intervalDays);
+            List<String> targetUrl = jdbcTemplateManager.getTemplate(target).queryForList(sql, String.class, taskName, intervalDays);
+
+            // 判断交集数据
+            List<String> intersection = new ArrayList<>(sourceUrl);
+            intersection.retainAll(targetUrl);
+            log.info("@@@ [{}] 记录数: [{}], [{}] 记录数: [{}], 交集条数: [{}]", source, sourceUrl.size(), target, targetUrl.size(), intersection.size());
+
+            // 以 source 为基准，判断差集数据
+            List<String> diff = findDifference(sourceUrl, intersection);
+            if (diff.size() > 0) {
+                log.info("@@@ [{}] 有，但是 [{}] 没有的记录数: {}", source, target, diff.size());
+                checkDiffReason(diff, jdbcTemplateManager.getTemplate(target));
+            }
+
+
+            // 以 target 为基准，判断差集数据
+            diff = findDifference(targetUrl, intersection);
+            if (diff.size() > 0) {
+                log.info("@@@ [{}] 有，但是 [{}] 没有的记录数: {}", target, source, diff.size());
+                checkDiffReason(diff, jdbcTemplateManager.getTemplate(source));
+            }
+
+
+            // 判断交集数据是否相同
+            findFieldsDiff(intersection, jdbcTemplateManager.getTemplate(source), jdbcTemplateManager.getTemplate(target));
+        }
+    }
+
+    private void findFieldsDiff(List<String> urls, JdbcTemplate jdbcTemplate, JdbcTemplate jdbcTemplate2) {
+        int count = 0;
+        Map<String, Map<String, List<Object>>> result = new HashMap<>();
+        String sql = "select * from all_news where url = ? limit 1";
+        for (String s : urls) {
+            if (urls.size() > helperProperties.getThreshold() && Math.random() > helperProperties.getSampleRate()) {
+                continue;
+            }
+            count++;
+            Map<String, Object> map = jdbcTemplate.queryForMap(sql, s);
+            Map<String, Object> map2 = jdbcTemplate2.queryForMap(sql, s);
+            Map<String, List<Object>> differentKeys = checkDifferentKeys(map, map2);
+            if (differentKeys.size() > 0) {
+                result.put(s, differentKeys);
+            }
+        }
+        if (ObjectUtils.isEmpty(result)) {
+            log.info("@@@ 在 [{}] 条中抽检了 [{}] 条，没有字段差异", urls.size(), count);
+        } else {
+
+            Set<String> keys = new HashSet<>();
+            for (Map<String, List<Object>> value : result.values()) {
+                keys.addAll(value.keySet());
+            }
+            log.warn("@@@ 共在 [{}] 条中抽检了 [{}] 条，有 [{}] 条有字段差异, 差异字段: {}", urls.size(), count, result.size(), keys);
+        }
+    }
+
+    public List<String> findDifference(List<String> source, List<String> intersection) {
+        List<String> diff = new ArrayList<>(source);
+        diff.removeAll(intersection);
+        return diff;
+    }
+
+    public void checkDiffReason(List<String> urls, JdbcTemplate jdbcTemplate) {
+        List<String> occupied = new ArrayList<>();
+        List<String> notExist = new ArrayList<>();
+
+        String sql = "select spider from all_news where url = ?";
+        for (String url : urls) {
+            List<String> spiders = jdbcTemplate.queryForList(sql, String.class, url);
+            if (spiders.size() == 1) {
+                occupied.add(url);
+            } else {
+                notExist.add(url);
+            }
+        }
+
+        if (!ObjectUtils.isEmpty(occupied)) {
+            log.info("@@ 已经被其他爬虫提前爬取的记录条数`: {}", occupied.size());
+        }
+        if (!ObjectUtils.isEmpty(notExist)) {
+            if (notExist.size() > 5) {
+                log.error("@@ ⚠️不存在的记录条数: {}", notExist.size());
+            } else {
+                log.warn("@@ 不存在的记录条数: {}, url: {}", notExist.size(), notExist);
+            }
+        }
+
+
+    }
+
+
+}

helper/src/main/java/work/xuye/helper/checker/NewsTypeEnum.java

@@ -0,0 +1,14 @@
+package work.xuye.helper.checker;
+
+
+/**
+ * @author xuye
+ * @since 2023/3/4
+ **/
+
+public enum NewsTypeEnum {
+
+    NEWS,
+    VIDEO
+}
+

helper/src/main/java/work/xuye/helper/controller/CheckController.java

@@ -0,0 +1,35 @@
+package work.xuye.helper.controller;
+
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
+import lombok.RequiredArgsConstructor;
+import org.springframework.validation.annotation.Validated;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+import work.xuye.helper.checker.DataDiffChecker;
+import work.xuye.helper.vo.DataCheckVO;
+
+import javax.validation.Valid;
+
+/**
+ * @author xuye
+ * @since 2023/7/13 18:13
+ **/
+@Api(tags = "数据对比")
+@Validated
+@RestController
+@RequestMapping("/check")
+@RequiredArgsConstructor
+public class CheckController {
+
+    private final DataDiffChecker dataDiffChecker;
+
+    @PostMapping
+    @ApiOperation("指定任务对比")
+    public void check(@RequestBody @Valid DataCheckVO dataCheckVO) {
+        dataDiffChecker.check(dataCheckVO.getIds(), dataCheckVO.getIntervalDays());
+    }
+
+}

helper/src/main/java/work/xuye/helper/properties/DatasourceConfigProperties.java

@@ -0,0 +1,28 @@
+package work.xuye.helper.properties;
+
+import lombok.Data;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+import java.util.Map;
+
+/**
+ * @author xuye
+ * @since 2023/1/14 21:50
+ **/
+@Data
+@Component
+@ConfigurationProperties(prefix = DatasourceConfigProperties.PROPERTIES_PREFIX)
+public class DatasourceConfigProperties {
+
+    public static final String PROPERTIES_PREFIX = "datasource";
+
+    private Map<String, DataSource> configMap;
+
+    @Data
+    public static class DataSource {
+        private String url;
+        private String username;
+        private String password;
+    }
+}

helper/src/main/java/work/xuye/helper/properties/HelperProperties.java

@@ -1,28 +1,46 @@
 package work.xuye.helper.properties;
 
 import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.boot.context.properties.ConfigurationProperties;
 import org.springframework.stereotype.Component;
+import work.xuye.helper.checker.NewsTypeEnum;
 
+import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 /**
  * @author xuye
  * @since 2023/1/14 21:50
  **/
+@Slf4j
 @Data
 @Component
 @ConfigurationProperties(prefix = HelperProperties.PROPERTIES_PREFIX)
 public class HelperProperties {
 
+
     public static final String PROPERTIES_PREFIX = "helper";
+    private String sourceDbName;
+    private String targetDbName;
+    private Map<NewsTypeEnum, List<Integer>> taskTypeIdMap;
+
+    /**
+     * 对比字段时，忽略的字段
+     */
+    private Set<String> equalsIgnoreField;
+
+
+    /**
+     * 触发抽检的阈值
+     */
+    private int threshold = 1000;
+
+    /**
+     * 抽检的比例
+     */
+    private double sampleRate = 0.1;
 
-    private Map<String, DataSource> datasourceMap;
 
-    @Data
-    public static class DataSource {
-        private String url;
-        private String username;
-        private String password;
-    }
 }

helper/src/main/java/work/xuye/helper/service/JdbcTemplateManager.java

@@ -0,0 +1,44 @@
+package work.xuye.helper.service;
+
+import com.zaxxer.hikari.HikariConfig;
+import com.zaxxer.hikari.HikariDataSource;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.stereotype.Service;
+import work.xuye.helper.properties.DatasourceConfigProperties;
+
+import javax.annotation.PostConstruct;
+import java.util.HashMap;
+
+/**
+ * @author xuye
+ * @since 2023/7/13 14:04
+ **/
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class JdbcTemplateManager {
+
+
+    private final DatasourceConfigProperties datasourceConfigProperties;
+
+    private final HashMap<String, JdbcTemplate> jdbcTemplateMap = new HashMap<>();
+
+    public JdbcTemplate getTemplate(String key) {
+        return jdbcTemplateMap.get(key);
+    }
+
+    @PostConstruct
+    private void initJdbcTemplate() {
+        jdbcTemplateMap.clear();
+        datasourceConfigProperties.getConfigMap().forEach((key, source) -> {
+            HikariConfig config = new HikariConfig();
+            config.setJdbcUrl(source.getUrl());
+            config.setUsername(source.getUsername());
+            config.setPassword(source.getPassword());
+            HikariDataSource dataSource = new HikariDataSource(config);
+            jdbcTemplateMap.put(key, new JdbcTemplate(dataSource));
+        });
+    }
+}

helper/src/main/java/work/xuye/helper/service/UpdateService.java

@@ -1,171 +0,0 @@
-package work.xuye.helper.service;
-
-import cn.hutool.db.Db;
-import cn.hutool.db.Entity;
-import com.zaxxer.hikari.HikariConfig;
-import com.zaxxer.hikari.HikariDataSource;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.boot.ApplicationArguments;
-import org.springframework.boot.ApplicationRunner;
-import org.springframework.stereotype.Service;
-import org.springframework.util.ObjectUtils;
-import work.xuye.helper.properties.HelperProperties;
-
-import javax.annotation.PostConstruct;
-import java.sql.SQLException;
-import java.util.*;
-
-/**
- * @author xuye
- * @since 2023/7/12 14:55
- **/
-@Slf4j
-@Service
-@RequiredArgsConstructor
-public class UpdateService implements ApplicationRunner {
-
-    private final HelperProperties helperProperties;
-
-    private final HashMap<String, HikariDataSource> dataSourceMap;
-
-    @PostConstruct
-    public void init() {
-
-        Map<String, HelperProperties.DataSource> datasourceConfigMap = helperProperties.getDatasourceMap();
-        for (String key : datasourceConfigMap.keySet()) {
-            HelperProperties.DataSource source = datasourceConfigMap.get(key);
-            HikariConfig config = new HikariConfig();
-            config.setJdbcUrl(source.getUrl());
-            config.setUsername(source.getUsername());
-            config.setPassword(source.getPassword());
-            HikariDataSource dataSource = new HikariDataSource(config);
-            dataSourceMap.put(key, dataSource);
-
-
-        }
-    }
-
-    public void start() throws SQLException {
-        String sql = "select date(createDate) as date, spider, count(*) as count\n" +
-                "from all_news\n" +
-                "where JSON_EXTRACT(raw_data, '$.crawler') = 'snp'\n" +
-                "  and createDate > curdate() - interval 1 day\n" +
-                "group by spider, date\n" +
-                "order by date desc, count desc;";
-
-
-        Set<String> keys = dataSourceMap.keySet();
-
-        HashMap<String, List<Entity>> map = new HashMap<>();
-
-        for (String key : keys) {
-            HikariDataSource dataSource = dataSourceMap.get(key);
-            List<Entity> query = Db.use(dataSourceMap.get(key)).query(sql);
-            map.put(key, query);
-        }
-
-        // Compare List<Entity> between different environments
-        for (String key1 : keys) {
-            for (String key2 : keys) {
-                if (!key1.equals(key2)) {
-                    List<Entity> list1 = map.get(key1);
-                    List<Entity> list2 = map.get(key2);
-
-                    for (Entity entity1 : list1) {
-                        for (Entity entity2 : list2) {
-                            String date1 = entity1.getStr("date");
-                            String spider1 = entity1.getStr("spider");
-                            int count1 = entity1.getInt("count");
-
-                            String date2 = entity2.getStr("date");
-                            String spider2 = entity2.getStr("spider");
-                            int count2 = entity2.getInt("count");
-
-                            if (date1.equals(date2) && spider1.equals(spider2) && count1 != count2) {
-                                int diff = Math.abs(count1 - count2);
-                                String more = count1 > count2 ? key1 : key2;
-                                System.out.printf("@@@ [%s] [%s] %s: %d %s: %d, diff abs: %d, more: %s%n", date1, spider1, key1, count1, key2, count2, diff, more);
-                                this.findDIff(spider1, date1);
-                            }
-                        }
-                    }
-                }
-            }
-            // todo 此处比较多个的话有遗漏，后续优化下
-            break;
-        }
-    }
-
-
-    public void findDIff(String spider, String date) throws SQLException {
-        HashMap<String, List<String>> map = new HashMap<>();
-        for (String key : dataSourceMap.keySet()) {
-            HikariDataSource dataSource = dataSourceMap.get(key);
-
-            String sql = "SELECT url FROM all_news WHERE spider = '" + spider + "' AND date(createDate) = '" + date + "';";
-            List<Entity> resultList = Db.use(dataSource).query(sql);
-            ArrayList<String> list = new ArrayList<>();
-            for (Entity entity : resultList) {
-                String url = entity.getStr("url");
-                list.add(url);
-            }
-            map.put(key, list);
-        }
-
-        for (String s : map.keySet()) {
-            this.compareEnvironments(map, s);
-        }
-
-
-    }
-
-    public void compareEnvironments(HashMap<String, List<String>> map, String environment) throws SQLException {
-
-        List<String> environmentList = map.get(environment);
-        if (environmentList == null) {
-            return;
-        }
-        for (String key : map.keySet()) {
-            if (key.equals(environment)) {
-                continue;
-            }
-            List<String> list = map.getOrDefault(key, new ArrayList<>());
-            List<String> intersection = new ArrayList<>(list);
-            intersection.retainAll(environmentList);
-
-            List<String> different = new ArrayList<>(list);
-            different.removeAll(intersection);
-
-            if (!different.isEmpty()) {
-                System.out.println("@@ 环境 " + environment + " 中，List " + key + " 中独有的元素有" + different.size() + "个");
-                for (String s : different) {
-                    getStatusByUrl(s);
-                }
-            }
-        }
-    }
-
-    public void getStatusByUrl(String url) throws SQLException {
-        String sql = "select spider from all_news where url='" + url + "';";
-        for (String key : dataSourceMap.keySet()) {
-            HikariDataSource dataSource = dataSourceMap.get(key);
-            List<Entity> resultList = Db.use(dataSource).query(sql);
-            if (ObjectUtils.isEmpty(resultList)) {
-                System.err.println("⚠️ url: " + url + " not found in " + key);
-            }
-            if (resultList.size() == 1) {
-                System.out.println("url [" + url + "] 被 [" + resultList.get(0).get("spider") + "] 提前抓取了");
-            }
-            if (resultList.size() > 1) {
-                System.err.println("!!! url: " + url + " found in " + key + " " + resultList.size() + " times");
-            }
-
-        }
-    }
-
-    @Override
-    public void run(ApplicationArguments args) throws Exception {
-        this.start();
-    }
-}

helper/src/main/java/work/xuye/helper/vo/DataCheckVO.java

@@ -0,0 +1,24 @@
+package work.xuye.helper.vo;
+
+import io.swagger.annotations.ApiModelProperty;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.List;
+
+/**
+ * @author xuye
+ * @since 2023/7/14 18:41
+ **/
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+public class DataCheckVO {
+
+    @ApiModelProperty("任务id")
+    private List<Integer> ids;
+
+    @ApiModelProperty("最近几天的数据")
+    private Integer intervalDays;
+}

helper/src/main/resources/application.yml

@@ -1,15 +1,48 @@
 knife4j:
   enable: true
 server:
-  port: 9300
-helper:
-  datasourceMap:
-    sit:
+  port: 9301
+datasource:
+  configMap:
+    news-sit:
       url: jdbc:mysql://47.116.58.10:3306/pyspider_sit_resultdb?useUnicode=true&characterEncoding=UTF-8&serverTimezone=GMT%2b8
       username: pyspider_user
       password: ND0328qSywfre
-    prd:
+    news-prd:
       url: jdbc:mysql://47.103.55.230:3306/pyspider_resultdb?characterEncoding=utf8&autoReconnect=true&useUnicode=true&useSSL=false
       username: pyspider
       password: strzsJQWp%uw9oKB
-
+    video-prd:
+      url: jdbc:mysql://47.116.61.180:3306/fhl_data_ingestion?useUnicode=true&characterEncoding=UTF-8&serverTimezone=GMT%2b8
+      username: fhl_owner
+      password: 5rDjEXN%bAq5Bf#9
+sq:
+  sentry:
+    enabled: false
+helper:
+  equals-ignore-field:
+    - id
+    - createDate
+    - postDate
+    - update_time
+  task-type-id-map:
+    news:
+      - 1
+      - 2
+      - 3
+      - 4
+      - 5
+      - 7
+      - 8
+      - 9
+      - 14
+      - 15
+      - 16
+      - 17
+    video:
+      - 6
+      - 10
+      - 11
+      - 12
+  source-db-name: news-sit
+  target-db-name: news-prd

source/src/main/java/work/xuye/source/properties/SourceProperties.java

@@ -19,7 +19,9 @@ public class SourceProperties {
 
     public static final String PROPERTIES_PREFIX = "request";
     private Pool pool;
-    private Integer timeoutSeconds = 30;
+    private Integer connectTimeoutSeconds = 20;
+    private Integer readTimeoutSeconds = 20;
+
 
     @Data
     @NoArgsConstructor

source/src/main/java/work/xuye/source/request/OkHttpRequestClient.java

@@ -38,15 +38,18 @@ public class OkHttpRequestClient implements RequestClient {
         this.gson = gson;
         SourceProperties.Pool poolConfig = sourceProperties.getPool();
         this.client = new OkHttpClient.Builder()
-                .connectTimeout(sourceProperties.getTimeoutSeconds(), TimeUnit.SECONDS)
+                .connectTimeout(sourceProperties.getConnectTimeoutSeconds(), TimeUnit.SECONDS)
+                .readTimeout(sourceProperties.getReadTimeoutSeconds(), TimeUnit.SECONDS)
                 .connectionPool(new ConnectionPool(
                         poolConfig.getMaxIdleConnections(),
                         poolConfig.getKeepAliveSeconds(),
                         TimeUnit.SECONDS))
                 .followRedirects(true)
                 .build();
-        log.info("init OkHttpRequestClient success, timeoutSeconds: {}, maxIdleConnections: {}, keepAliveSeconds: {}",
-                sourceProperties.getTimeoutSeconds(),
+
+        log.info("init OkHttpRequestClient success, connectTimeoutSeconds: {}, readTimeoutSeconds: {}, maxIdleConnections: {}, keepAliveSeconds: {}",
+                sourceProperties.getConnectTimeoutSeconds(),
+                sourceProperties.getReadTimeoutSeconds(),
                 poolConfig.getMaxIdleConnections(),
                 poolConfig.getKeepAliveSeconds());
     }
@@ -98,8 +101,10 @@ public class OkHttpRequestClient implements RequestClient {
         // res
         HttpRes httpRes = new HttpRes();
         Request httpRequest = requestBuilder.build();
+        httpRes.requestParams(request);
         try (Response response = client.newCall(httpRequest).execute()) {
-            httpRes.requestParams(request).headers(response.headers().toMultimap()).status(HttpStatus.valueOf(response.code()));
+            httpRes.headers(response.headers().toMultimap())
+                    .status(HttpStatus.valueOf(response.code()));
             if (!ObjectUtils.isEmpty(response.body())) {
                 BufferedSource source = response.body().source();
                 Charset charset = CharsetUtil.getCharsetByName(request.getCharset());

transformer/src/main/java/work/xuye/transformer/transformer/XmlToJsonTransformer.java

@@ -14,7 +14,7 @@ import org.springframework.util.ObjectUtils;
 @Slf4j
 @Component("xml2json")
 public class XmlToJsonTransformer implements MessageTransformer {
-    
+
     @Override
     public String transform(String xml, String seedUrl) {
         if (ObjectUtils.isEmpty(xml)) {
@@ -24,7 +24,7 @@ public class XmlToJsonTransformer implements MessageTransformer {
         try {
             jsonObject = XML.toJSONObject(xml);
         } catch (JSONException e) {
-            log.info("xml2json transform error, xml:{}, seedUrl:{}, exception:{}", xml, seedUrl, e);
+            log.warn("xml2json transform error,xml:{},seedUrl:{},error:{}", xml, seedUrl, e.getMessage());
             throw new RuntimeException("xml2json transform error, seedUrl:" + seedUrl);
         }
         return jsonObject.toString();