From 58faf166b63d4d8e36c5f7880325c7d73fa840e0 Mon Sep 17 00:00:00 2001 From: xuye Date: Thu, 13 Jul 2023 18:09:37 +0800 Subject: [PATCH] dev --- common/pom.xml | 7 +- .../work/xuye/common/service/MessageService.java | 2 +- .../work/xuye/helper/checker/DataDiffChecker.java | 180 +++++++++++++++++++++ .../work/xuye/helper/checker/NewsTypeEnum.java | 14 ++ .../xuye/helper/controller/CheckController.java | 35 ++++ .../properties/DatasourceConfigProperties.java | 28 ++++ .../xuye/helper/properties/HelperProperties.java | 32 +++- .../xuye/helper/service/JdbcTemplateManager.java | 44 +++++ .../work/xuye/helper/service/UpdateService.java | 171 -------------------- .../main/java/work/xuye/helper/vo/DataCheckVO.java | 24 +++ helper/src/main/resources/application.yml | 45 +++++- .../xuye/source/properties/SourceProperties.java | 4 +- .../xuye/source/request/OkHttpRequestClient.java | 13 +- .../transformer/XmlToJsonTransformer.java | 4 +- 14 files changed, 410 insertions(+), 193 deletions(-) create mode 100644 helper/src/main/java/work/xuye/helper/checker/DataDiffChecker.java create mode 100644 helper/src/main/java/work/xuye/helper/checker/NewsTypeEnum.java create mode 100644 helper/src/main/java/work/xuye/helper/controller/CheckController.java create mode 100644 helper/src/main/java/work/xuye/helper/properties/DatasourceConfigProperties.java create mode 100644 helper/src/main/java/work/xuye/helper/service/JdbcTemplateManager.java delete mode 100644 helper/src/main/java/work/xuye/helper/service/UpdateService.java create mode 100644 helper/src/main/java/work/xuye/helper/vo/DataCheckVO.java diff --git a/common/pom.xml b/common/pom.xml index d108c42..50d9ee6 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -1,5 +1,6 @@ - + 4.0.0 work.xuye @@ -120,6 +121,10 @@ sq-sentry 1.1.8 + + org.springframework.boot + spring-boot-starter-validation + diff --git a/common/src/main/java/work/xuye/common/service/MessageService.java b/common/src/main/java/work/xuye/common/service/MessageService.java index a7ded8d..0f94e13 100644 --- a/common/src/main/java/work/xuye/common/service/MessageService.java +++ b/common/src/main/java/work/xuye/common/service/MessageService.java @@ -58,7 +58,7 @@ public class MessageService { .send(); messageMD5Store.save(result.toString()); } else { - log.debug("短期内已经发送过相同的异常信息，不再发送: [{}]", result); + log.info("短期内已经发送过相同的异常信息，不再发送: [{}]", result); } } } diff --git a/helper/src/main/java/work/xuye/helper/checker/DataDiffChecker.java b/helper/src/main/java/work/xuye/helper/checker/DataDiffChecker.java new file mode 100644 index 0000000..4067cd1 --- /dev/null +++ b/helper/src/main/java/work/xuye/helper/checker/DataDiffChecker.java @@ -0,0 +1,180 @@ +package work.xuye.helper.checker; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Service; +import org.springframework.util.ObjectUtils; +import work.xuye.common.db.service.TaskManager; +import work.xuye.helper.properties.HelperProperties; +import work.xuye.helper.service.JdbcTemplateManager; + +import java.util.*; + +/** + * @author xuye + * @since 2023/7/13 14:57 + **/ +@Slf4j +@Service +@RequiredArgsConstructor +public class DataDiffChecker { + + private final TaskManager taskManager; + private final JdbcTemplateManager jdbcTemplateManager; + private final HelperProperties helperProperties; + + + public Map> checkDifferentKeys(Map map1, Map map2) { + Map> map = new HashMap<>(); + for (String key : map1.keySet()) { + if (map2.containsKey(key)) { + Object value1 = map1.get(key); + Object value2 = map2.get(key); + + if (!ObjectUtils.nullSafeEquals(value1, value2)) { + if (!helperProperties.getEqualsIgnoreField().contains(key)) { + map.put(key, List.of(value1, value2)); + log.debug("@@@ key: {}, value1: {}, value2: {}", key, value1, value2); + } + } + } else { + log.debug("@@@ key: {}, value1: {}, value2: {}", key, map1.get(key), null); + ArrayList list = new ArrayList<>(); + list.add(map1.get(key)); + list.add(null); + map.put(key, list); + } + } + for (String key : map2.keySet()) { + if (!map1.containsKey(key)) { + log.debug("@@@ key: {}, value1: {}, value2: {}", key, null, map2.get(key)); + ArrayList list = new ArrayList<>(); + list.add(null); + list.add(map2.get(key)); + map.put(key, list); + } + } + return map; + } + + + public void check(List ids, Integer intervalDays) { + List newsIds = helperProperties.getTaskTypeIdMap().get(NewsTypeEnum.NEWS); + List videoIds = helperProperties.getTaskTypeIdMap().get(NewsTypeEnum.VIDEO); + if (ObjectUtils.isEmpty(ids)) { + handleNews(newsIds, intervalDays); +// handleVideo(videoIds); + } else { + for (Integer id : ids) { + if (newsIds.contains(id)) { + handleNews(List.of(id), intervalDays); + } else if (videoIds.contains(id)) { +// handleVideo(List.of(id)); + } + } + } + } + + public void handleNews(List ids, Integer intervalDays) { + + String source = helperProperties.getSourceDbName(); + String target = helperProperties.getTargetDbName(); + + String sql = "select url from all_news where spider = ? and createDate > CURDATE() - INTERVAL ? DAY order by createDate desc"; + for (Integer id : ids) { + String taskName = taskManager.getTaskInfoByTaskId(id).getTask().getName(); + log.info("-------------------------{}-------------------------", taskName); + List sourceUrl = jdbcTemplateManager.getTemplate(source).queryForList(sql, String.class, taskName, intervalDays); + List targetUrl = jdbcTemplateManager.getTemplate(target).queryForList(sql, String.class, taskName, intervalDays); + + // 判断交集数据 + List intersection = new ArrayList<>(sourceUrl); + intersection.retainAll(targetUrl); + log.info("@@@ [{}] 记录数: [{}], [{}] 记录数: [{}], 交集条数: [{}]", source, sourceUrl.size(), target, targetUrl.size(), intersection.size()); + + // 以 source 为基准，判断差集数据 + List diff = findDifference(sourceUrl, intersection); + if (diff.size() > 0) { + log.info("@@@ [{}] 有，但是 [{}] 没有的记录数: {}", source, target, diff.size()); + checkDiffReason(diff, jdbcTemplateManager.getTemplate(target)); + } + + + // 以 target 为基准，判断差集数据 + diff = findDifference(targetUrl, intersection); + if (diff.size() > 0) { + log.info("@@@ [{}] 有，但是 [{}] 没有的记录数: {}", target, source, diff.size()); + checkDiffReason(diff, jdbcTemplateManager.getTemplate(source)); + } + + + // 判断交集数据是否相同 + findFieldsDiff(intersection, jdbcTemplateManager.getTemplate(source), jdbcTemplateManager.getTemplate(target)); + } + } + + private void findFieldsDiff(List urls, JdbcTemplate jdbcTemplate, JdbcTemplate jdbcTemplate2) { + int count = 0; + Map>> result = new HashMap<>(); + String sql = "select * from all_news where url = ? limit 1"; + for (String s : urls) { + if (urls.size() > helperProperties.getThreshold() && Math.random() > helperProperties.getSampleRate()) { + continue; + } + count++; + Map map = jdbcTemplate.queryForMap(sql, s); + Map map2 = jdbcTemplate2.queryForMap(sql, s); + Map> differentKeys = checkDifferentKeys(map, map2); + if (differentKeys.size() > 0) { + result.put(s, differentKeys); + } + } + if (ObjectUtils.isEmpty(result)) { + log.info("@@@ 在 [{}] 条中抽检了 [{}] 条，没有字段差异", urls.size(), count); + } else { + + Set keys = new HashSet<>(); + for (Map> value : result.values()) { + keys.addAll(value.keySet()); + } + log.warn("@@@ 共在 [{}] 条中抽检了 [{}] 条，有 [{}] 条有字段差异, 差异字段: {}", urls.size(), count, result.size(), keys); + } + } + + public List findDifference(List source, List intersection) { + List diff = new ArrayList<>(source); + diff.removeAll(intersection); + return diff; + } + + public void checkDiffReason(List urls, JdbcTemplate jdbcTemplate) { + List occupied = new ArrayList<>(); + List notExist = new ArrayList<>(); + + String sql = "select spider from all_news where url = ?"; + for (String url : urls) { + List spiders = jdbcTemplate.queryForList(sql, String.class, url); + if (spiders.size() == 1) { + occupied.add(url); + } else { + notExist.add(url); + } + } + + if (!ObjectUtils.isEmpty(occupied)) { + log.info("@@ 已经被其他爬虫提前爬取的记录条数`: {}", occupied.size()); + } + if (!ObjectUtils.isEmpty(notExist)) { + if (notExist.size() > 5) { + log.error("@@ ⚠️不存在的记录条数: {}", notExist.size()); + } else { + log.warn("@@ 不存在的记录条数: {}, url: {}", notExist.size(), notExist); + } + } + + + } + + +} diff --git a/helper/src/main/java/work/xuye/helper/checker/NewsTypeEnum.java b/helper/src/main/java/work/xuye/helper/checker/NewsTypeEnum.java new file mode 100644 index 0000000..ac2fcce --- /dev/null +++ b/helper/src/main/java/work/xuye/helper/checker/NewsTypeEnum.java @@ -0,0 +1,14 @@ +package work.xuye.helper.checker; + + +/** + * @author xuye + * @since 2023/3/4 + **/ + +public enum NewsTypeEnum { + + NEWS, + VIDEO +} + diff --git a/helper/src/main/java/work/xuye/helper/controller/CheckController.java b/helper/src/main/java/work/xuye/helper/controller/CheckController.java new file mode 100644 index 0000000..8a8c5f5 --- /dev/null +++ b/helper/src/main/java/work/xuye/helper/controller/CheckController.java @@ -0,0 +1,35 @@ +package work.xuye.helper.controller; + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import lombok.RequiredArgsConstructor; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import work.xuye.helper.checker.DataDiffChecker; +import work.xuye.helper.vo.DataCheckVO; + +import javax.validation.Valid; + +/** + * @author xuye + * @since 2023/7/13 18:13 + **/ +@Api(tags = "数据对比") +@Validated +@RestController +@RequestMapping("/check") +@RequiredArgsConstructor +public class CheckController { + + private final DataDiffChecker dataDiffChecker; + + @PostMapping + @ApiOperation("指定任务对比") + public void check(@RequestBody @Valid DataCheckVO dataCheckVO) { + dataDiffChecker.check(dataCheckVO.getIds(), dataCheckVO.getIntervalDays()); + } + +} diff --git a/helper/src/main/java/work/xuye/helper/properties/DatasourceConfigProperties.java b/helper/src/main/java/work/xuye/helper/properties/DatasourceConfigProperties.java new file mode 100644 index 0000000..4aa0014 --- /dev/null +++ b/helper/src/main/java/work/xuye/helper/properties/DatasourceConfigProperties.java @@ -0,0 +1,28 @@ +package work.xuye.helper.properties; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +import java.util.Map; + +/** + * @author xuye + * @since 2023/1/14 21:50 + **/ +@Data +@Component +@ConfigurationProperties(prefix = DatasourceConfigProperties.PROPERTIES_PREFIX) +public class DatasourceConfigProperties { + + public static final String PROPERTIES_PREFIX = "datasource"; + + private Map configMap; + + @Data + public static class DataSource { + private String url; + private String username; + private String password; + } +} diff --git a/helper/src/main/java/work/xuye/helper/properties/HelperProperties.java b/helper/src/main/java/work/xuye/helper/properties/HelperProperties.java index e91b50e..9ec8ae7 100644 --- a/helper/src/main/java/work/xuye/helper/properties/HelperProperties.java +++ b/helper/src/main/java/work/xuye/helper/properties/HelperProperties.java @@ -1,28 +1,46 @@ package work.xuye.helper.properties; import lombok.Data; +import lombok.extern.slf4j.Slf4j; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; +import work.xuye.helper.checker.NewsTypeEnum; +import java.util.List; import java.util.Map; +import java.util.Set; /** * @author xuye * @since 2023/1/14 21:50 **/ +@Slf4j @Data @Component @ConfigurationProperties(prefix = HelperProperties.PROPERTIES_PREFIX) public class HelperProperties { + public static final String PROPERTIES_PREFIX = "helper"; + private String sourceDbName; + private String targetDbName; + private Map> taskTypeIdMap; + + /** + * 对比字段时，忽略的字段 + */ + private Set equalsIgnoreField; + + + /** + * 触发抽检的阈值 + */ + private int threshold = 1000; + + /** + * 抽检的比例 + */ + private double sampleRate = 0.1; - private Map datasourceMap; - @Data - public static class DataSource { - private String url; - private String username; - private String password; - } } diff --git a/helper/src/main/java/work/xuye/helper/service/JdbcTemplateManager.java b/helper/src/main/java/work/xuye/helper/service/JdbcTemplateManager.java new file mode 100644 index 0000000..05913d3 --- /dev/null +++ b/helper/src/main/java/work/xuye/helper/service/JdbcTemplateManager.java @@ -0,0 +1,44 @@ +package work.xuye.helper.service; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Service; +import work.xuye.helper.properties.DatasourceConfigProperties; + +import javax.annotation.PostConstruct; +import java.util.HashMap; + +/** + * @author xuye + * @since 2023/7/13 14:04 + **/ +@Slf4j +@Service +@RequiredArgsConstructor +public class JdbcTemplateManager { + + + private final DatasourceConfigProperties datasourceConfigProperties; + + private final HashMap jdbcTemplateMap = new HashMap<>(); + + public JdbcTemplate getTemplate(String key) { + return jdbcTemplateMap.get(key); + } + + @PostConstruct + private void initJdbcTemplate() { + jdbcTemplateMap.clear(); + datasourceConfigProperties.getConfigMap().forEach((key, source) -> { + HikariConfig config = new HikariConfig(); + config.setJdbcUrl(source.getUrl()); + config.setUsername(source.getUsername()); + config.setPassword(source.getPassword()); + HikariDataSource dataSource = new HikariDataSource(config); + jdbcTemplateMap.put(key, new JdbcTemplate(dataSource)); + }); + } +} diff --git a/helper/src/main/java/work/xuye/helper/service/UpdateService.java b/helper/src/main/java/work/xuye/helper/service/UpdateService.java deleted file mode 100644 index 5f3b138..0000000 --- a/helper/src/main/java/work/xuye/helper/service/UpdateService.java +++ /dev/null @@ -1,171 +0,0 @@ -package work.xuye.helper.service; - -import cn.hutool.db.Db; -import cn.hutool.db.Entity; -import com.zaxxer.hikari.HikariConfig; -import com.zaxxer.hikari.HikariDataSource; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.ApplicationArguments; -import org.springframework.boot.ApplicationRunner; -import org.springframework.stereotype.Service; -import org.springframework.util.ObjectUtils; -import work.xuye.helper.properties.HelperProperties; - -import javax.annotation.PostConstruct; -import java.sql.SQLException; -import java.util.*; - -/** - * @author xuye - * @since 2023/7/12 14:55 - **/ -@Slf4j -@Service -@RequiredArgsConstructor -public class UpdateService implements ApplicationRunner { - - private final HelperProperties helperProperties; - - private final HashMap dataSourceMap; - - @PostConstruct - public void init() { - - Map datasourceConfigMap = helperProperties.getDatasourceMap(); - for (String key : datasourceConfigMap.keySet()) { - HelperProperties.DataSource source = datasourceConfigMap.get(key); - HikariConfig config = new HikariConfig(); - config.setJdbcUrl(source.getUrl()); - config.setUsername(source.getUsername()); - config.setPassword(source.getPassword()); - HikariDataSource dataSource = new HikariDataSource(config); - dataSourceMap.put(key, dataSource); - - - } - } - - public void start() throws SQLException { - String sql = "select date(createDate) as date, spider, count(*) as count\n" + - "from all_news\n" + - "where JSON_EXTRACT(raw_data, '$.crawler') = 'snp'\n" + - " and createDate > curdate() - interval 1 day\n" + - "group by spider, date\n" + - "order by date desc, count desc;"; - - - Set keys = dataSourceMap.keySet(); - - HashMap> map = new HashMap<>(); - - for (String key : keys) { - HikariDataSource dataSource = dataSourceMap.get(key); - List query = Db.use(dataSourceMap.get(key)).query(sql); - map.put(key, query); - } - - // Compare List between different environments - for (String key1 : keys) { - for (String key2 : keys) { - if (!key1.equals(key2)) { - List list1 = map.get(key1); - List list2 = map.get(key2); - - for (Entity entity1 : list1) { - for (Entity entity2 : list2) { - String date1 = entity1.getStr("date"); - String spider1 = entity1.getStr("spider"); - int count1 = entity1.getInt("count"); - - String date2 = entity2.getStr("date"); - String spider2 = entity2.getStr("spider"); - int count2 = entity2.getInt("count"); - - if (date1.equals(date2) && spider1.equals(spider2) && count1 != count2) { - int diff = Math.abs(count1 - count2); - String more = count1 > count2 ? key1 : key2; - System.out.printf("@@@ [%s] [%s] %s: %d %s: %d, diff abs: %d, more: %s%n", date1, spider1, key1, count1, key2, count2, diff, more); - this.findDIff(spider1, date1); - } - } - } - } - } - // todo 此处比较多个的话有遗漏，后续优化下 - break; - } - } - - - public void findDIff(String spider, String date) throws SQLException { - HashMap> map = new HashMap<>(); - for (String key : dataSourceMap.keySet()) { - HikariDataSource dataSource = dataSourceMap.get(key); - - String sql = "SELECT url FROM all_news WHERE spider = '" + spider + "' AND date(createDate) = '" + date + "';"; - List resultList = Db.use(dataSource).query(sql); - ArrayList list = new ArrayList<>(); - for (Entity entity : resultList) { - String url = entity.getStr("url"); - list.add(url); - } - map.put(key, list); - } - - for (String s : map.keySet()) { - this.compareEnvironments(map, s); - } - - - } - - public void compareEnvironments(HashMap> map, String environment) throws SQLException { - - List environmentList = map.get(environment); - if (environmentList == null) { - return; - } - for (String key : map.keySet()) { - if (key.equals(environment)) { - continue; - } - List list = map.getOrDefault(key, new ArrayList<>()); - List intersection = new ArrayList<>(list); - intersection.retainAll(environmentList); - - List different = new ArrayList<>(list); - different.removeAll(intersection); - - if (!different.isEmpty()) { - System.out.println("@@ 环境 " + environment + " 中，List " + key + " 中独有的元素有" + different.size() + "个"); - for (String s : different) { - getStatusByUrl(s); - } - } - } - } - - public void getStatusByUrl(String url) throws SQLException { - String sql = "select spider from all_news where url='" + url + "';"; - for (String key : dataSourceMap.keySet()) { - HikariDataSource dataSource = dataSourceMap.get(key); - List resultList = Db.use(dataSource).query(sql); - if (ObjectUtils.isEmpty(resultList)) { - System.err.println("⚠️ url: " + url + " not found in " + key); - } - if (resultList.size() == 1) { - System.out.println("url [" + url + "] 被 [" + resultList.get(0).get("spider") + "] 提前抓取了"); - } - if (resultList.size() > 1) { - System.err.println("!!! url: " + url + " found in " + key + " " + resultList.size() + " times"); - } - - } - } - - @Override - public void run(ApplicationArguments args) throws Exception { - this.start(); - } -} diff --git a/helper/src/main/java/work/xuye/helper/vo/DataCheckVO.java b/helper/src/main/java/work/xuye/helper/vo/DataCheckVO.java new file mode 100644 index 0000000..e56ac0f --- /dev/null +++ b/helper/src/main/java/work/xuye/helper/vo/DataCheckVO.java @@ -0,0 +1,24 @@ +package work.xuye.helper.vo; + +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +/** + * @author xuye + * @since 2023/7/14 18:41 + **/ +@Data +@NoArgsConstructor +@AllArgsConstructor +public class DataCheckVO { + + @ApiModelProperty("任务id") + private List ids; + + @ApiModelProperty("最近几天的数据") + private Integer intervalDays; +} diff --git a/helper/src/main/resources/application.yml b/helper/src/main/resources/application.yml index 30d5b4d..1210186 100644 --- a/helper/src/main/resources/application.yml +++ b/helper/src/main/resources/application.yml @@ -1,15 +1,48 @@ knife4j: enable: true server: - port: 9300 -helper: - datasourceMap: - sit: + port: 9301 +datasource: + configMap: + news-sit: url: jdbc:mysql://47.116.58.10:3306/pyspider_sit_resultdb?useUnicode=true&characterEncoding=UTF-8&serverTimezone=GMT%2b8 username: pyspider_user password: ND0328qSywfre - prd: + news-prd: url: jdbc:mysql://47.103.55.230:3306/pyspider_resultdb?characterEncoding=utf8&autoReconnect=true&useUnicode=true&useSSL=false username: pyspider password: strzsJQWp%uw9oKB - + video-prd: + url: jdbc:mysql://47.116.61.180:3306/fhl_data_ingestion?useUnicode=true&characterEncoding=UTF-8&serverTimezone=GMT%2b8 + username: fhl_owner + password: 5rDjEXN%bAq5Bf#9 +sq: + sentry: + enabled: false +helper: + equals-ignore-field: + - id + - createDate + - postDate + - update_time + task-type-id-map: + news: + - 1 + - 2 + - 3 + - 4 + - 5 + - 7 + - 8 + - 9 + - 14 + - 15 + - 16 + - 17 + video: + - 6 + - 10 + - 11 + - 12 + source-db-name: news-sit + target-db-name: news-prd \ No newline at end of file diff --git a/source/src/main/java/work/xuye/source/properties/SourceProperties.java b/source/src/main/java/work/xuye/source/properties/SourceProperties.java index 9490088..721e1e0 100644 --- a/source/src/main/java/work/xuye/source/properties/SourceProperties.java +++ b/source/src/main/java/work/xuye/source/properties/SourceProperties.java @@ -19,7 +19,9 @@ public class SourceProperties { public static final String PROPERTIES_PREFIX = "request"; private Pool pool; - private Integer timeoutSeconds = 30; + private Integer connectTimeoutSeconds = 20; + private Integer readTimeoutSeconds = 20; + @Data @NoArgsConstructor diff --git a/source/src/main/java/work/xuye/source/request/OkHttpRequestClient.java b/source/src/main/java/work/xuye/source/request/OkHttpRequestClient.java index 52ab1ed..9e5c3f2 100644 --- a/source/src/main/java/work/xuye/source/request/OkHttpRequestClient.java +++ b/source/src/main/java/work/xuye/source/request/OkHttpRequestClient.java @@ -38,15 +38,18 @@ public class OkHttpRequestClient implements RequestClient { this.gson = gson; SourceProperties.Pool poolConfig = sourceProperties.getPool(); this.client = new OkHttpClient.Builder() - .connectTimeout(sourceProperties.getTimeoutSeconds(), TimeUnit.SECONDS) + .connectTimeout(sourceProperties.getConnectTimeoutSeconds(), TimeUnit.SECONDS) + .readTimeout(sourceProperties.getReadTimeoutSeconds(), TimeUnit.SECONDS) .connectionPool(new ConnectionPool( poolConfig.getMaxIdleConnections(), poolConfig.getKeepAliveSeconds(), TimeUnit.SECONDS)) .followRedirects(true) .build(); - log.info("init OkHttpRequestClient success, timeoutSeconds: {}, maxIdleConnections: {}, keepAliveSeconds: {}", - sourceProperties.getTimeoutSeconds(), + + log.info("init OkHttpRequestClient success, connectTimeoutSeconds: {}, readTimeoutSeconds: {}, maxIdleConnections: {}, keepAliveSeconds: {}", + sourceProperties.getConnectTimeoutSeconds(), + sourceProperties.getReadTimeoutSeconds(), poolConfig.getMaxIdleConnections(), poolConfig.getKeepAliveSeconds()); } @@ -98,8 +101,10 @@ public class OkHttpRequestClient implements RequestClient { // res HttpRes httpRes = new HttpRes(); Request httpRequest = requestBuilder.build(); + httpRes.requestParams(request); try (Response response = client.newCall(httpRequest).execute()) { - httpRes.requestParams(request).headers(response.headers().toMultimap()).status(HttpStatus.valueOf(response.code())); + httpRes.headers(response.headers().toMultimap()) + .status(HttpStatus.valueOf(response.code())); if (!ObjectUtils.isEmpty(response.body())) { BufferedSource source = response.body().source(); Charset charset = CharsetUtil.getCharsetByName(request.getCharset()); diff --git a/transformer/src/main/java/work/xuye/transformer/transformer/XmlToJsonTransformer.java b/transformer/src/main/java/work/xuye/transformer/transformer/XmlToJsonTransformer.java index 2200a64..eb08226 100644 --- a/transformer/src/main/java/work/xuye/transformer/transformer/XmlToJsonTransformer.java +++ b/transformer/src/main/java/work/xuye/transformer/transformer/XmlToJsonTransformer.java @@ -14,7 +14,7 @@ import org.springframework.util.ObjectUtils; @Slf4j @Component("xml2json") public class XmlToJsonTransformer implements MessageTransformer { - + @Override public String transform(String xml, String seedUrl) { if (ObjectUtils.isEmpty(xml)) { @@ -24,7 +24,7 @@ public class XmlToJsonTransformer implements MessageTransformer { try { jsonObject = XML.toJSONObject(xml); } catch (JSONException e) { - log.info("xml2json transform error, xml:{}, seedUrl:{}, exception:{}", xml, seedUrl, e); + log.warn("xml2json transform error,xml:{},seedUrl:{},error:{}", xml, seedUrl, e.getMessage()); throw new RuntimeException("xml2json transform error, seedUrl:" + seedUrl); } return jsonObject.toString();