Merge remote-tracking branch 'origin/dev' into dev

liquidnet-bus-common/liquidnet-common-base/src/main/java/com/liquidnet/commons/lang/util/HttpUtil.java

 package com.liquidnet.commons.lang.util;
 
+import com.liquidnet.commons.lang.util.spring.RestTemplateConfig;
 import org.springframework.http.*;
 import org.springframework.util.LinkedMultiValueMap;
 import org.springframework.util.MultiValueMap;
@@ -14,7 +15,7 @@ public class HttpUtil {
     private static RestTemplate restTemplate;
 
     static {
-        restTemplate = new RestTemplate();
+        restTemplate = RestTemplateConfig.getRestTemplate();
     }
 
     /**

liquidnet-bus-common/liquidnet-common-base/src/main/java/com/liquidnet/commons/lang/util/spring/RestTemplateConfig.java

+package com.liquidnet.commons.lang.util.spring;
+
+import org.apache.http.client.HttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
+import org.springframework.web.client.DefaultResponseErrorHandler;
+import org.springframework.web.client.RestTemplate;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * @author AnJiabin <anjiabin@zhengzai.tv>
+ * @version V1.0
+ * @Description: TODO
+ * @class: RestTemplateConfig
+ * @Package com.liquidnet.commons.lang.util.spring
+ * @Copyright: LightNet @ Copyright (c) 2021
+ * @date 2021/7/31 14:57
+ */
+public class RestTemplateConfig {
+    private static RestTemplate restTemplate;
+
+    static {
+        // 长链接保持时间长度20秒
+        PoolingHttpClientConnectionManager poolConnManager =
+                new PoolingHttpClientConnectionManager(20, TimeUnit.SECONDS);
+        // 设置最大链接数
+        poolConnManager.setMaxTotal(500*getMaxCpuCore() + 500 );
+        // 单路由的并发数
+        poolConnManager.setDefaultMaxPerRoute(500*getMaxCpuCore());
+
+        HttpClientBuilder httpClientBuilder = HttpClients.custom();
+        httpClientBuilder.setConnectionManager(poolConnManager);
+
+        // 重试次数3次，并开启
+//        httpClientBuilder.setRetryHandler(new DefaultHttpRequestRetryHandler(3,true));
+        HttpClient httpClient = httpClientBuilder.build();
+//        // 保持长链接配置，keep-alive
+//        httpClientBuilder.setKeepAliveStrategy(new DefaultConnectionKeepAliveStrategy());
+
+        HttpComponentsClientHttpRequestFactory httpComponentsClientHttpRequestFactory = new HttpComponentsClientHttpRequestFactory(httpClient);
+
+        // 链接超时配置 5秒
+        httpComponentsClientHttpRequestFactory.setConnectTimeout(5000);
+        // 连接读取超时配置
+//        httpComponentsClientHttpRequestFactory.setReadTimeout(10000);
+        // 连接池不够用时候等待时间长度设置，分词那边 500毫秒 ，我们这边设置成1秒
+        httpComponentsClientHttpRequestFactory.setConnectionRequestTimeout(5000);
+
+        // 缓冲请求数据，POST大量数据，可以设定为true 我们这边机器比较内存较大
+        httpComponentsClientHttpRequestFactory.setBufferRequestBody(true);
+
+        restTemplate = new RestTemplate();
+        restTemplate.setRequestFactory(httpComponentsClientHttpRequestFactory);
+        restTemplate.setErrorHandler(new DefaultResponseErrorHandler());
+    }
+
+    public static RestTemplate getRestTemplate(){
+        return restTemplate;
+    }
+
+    private static int getMaxCpuCore(){
+        int cpuCore = Runtime.getRuntime().availableProcessors();
+        return  cpuCore;
+    }
+}

liquidnet-bus-service/liquidnet-service-kylin/liquidnet-service-kylin-impl/docu/mongo_db_del.sql

@@ -15,4 +15,4 @@ db.KylinTicketTimesPartnerVo.deleteMany({});
 db.PerformanceMemberAuditParam.deleteMany({});
 db.AdminUpushVo.deleteMany({});
 
-
+-- db.KylinOrderTicketVo.deleteMany({createdAt:/^2021-04.*/})

liquidnet-bus-service/liquidnet-service-platform/liquidnet-service-platform-impl/src/main/java/com/liquidnet/service/platform/service/impl/adam/dm/processor/DMAddressesProcessor.java

@@ -104,6 +104,10 @@ public class DMAddressesProcessor extends DataMigrationProcessorService {
                         addressesList.clear();
                     }
                 }
+                if (addressesList.size() > 0 && ct > 1000) {
+                    tl += addressesList.size();
+                    log.info("DM.execute.limit.result:{}", dmAddressesService.saveBatch(addressesList));
+                }
                 row.close();
                 statement.close();
 

liquidnet-bus-service/liquidnet-service-platform/liquidnet-service-platform-impl/src/main/java/com/liquidnet/service/platform/service/impl/adam/dm/processor/DMCollectionProcessor.java

@@ -95,6 +95,10 @@ public class DMCollectionProcessor extends DataMigrationProcessorService {
                         collectionList.clear();
                     }
                 }
+                if (collectionList.size() > 0 && ct > 1000) {
+                    tl += collectionList.size();
+                    log.info("DM.execute.limit.result:{}", dmCollectionService.saveBatch(collectionList));
+                }
                 row.close();
                 statement.close();
 

liquidnet-bus-service/liquidnet-service-platform/liquidnet-service-platform-impl/src/main/java/com/liquidnet/service/platform/service/impl/adam/dm/processor/DMEntersProcessor.java

@@ -108,6 +108,10 @@ public class DMEntersProcessor extends DataMigrationProcessorService {
                         entersList.clear();
                     }
                 }
+                if (entersList.size() > 0 && ct > 1000) {
+                    tl += entersList.size();
+                    log.info("DM.execute.limit.result:{}", dmEntersService.saveBatch(entersList));
+                }
                 row.close();
                 statement.close();
 

liquidnet-bus-service/liquidnet-service-platform/liquidnet-service-platform-impl/src/main/java/com/liquidnet/service/platform/service/impl/adam/dm/processor/DMMemberCodeProcessor.java

@@ -259,6 +259,10 @@ public class DMMemberCodeProcessor extends DataMigrationProcessorService {
                         memberCodeList.clear();
                     }
                 }
+                if (memberCodeList.size() > 0 && ct > 1000) {
+                    tl += memberCodeList.size();
+                    log.info("DM.execute.limit.result:{}", dmMemberCodeService.saveBatch(memberCodeList));
+                }
                 row.close();
                 statement.close();
 

liquidnet-bus-service/liquidnet-service-platform/liquidnet-service-platform-impl/src/main/java/com/liquidnet/service/platform/service/impl/adam/dm/processor/DMMemberOrderProcessor.java

@@ -140,6 +140,10 @@ public class DMMemberOrderProcessor extends DataMigrationProcessorService {
                         memberOrderList.clear();
                     }
                 }
+                if (memberOrderList.size() > 0 && ct > 1000) {
+                    tl += memberOrderList.size();
+                    log.info("DM.execute.limit.result:{}", dmMemberOrderService.saveBatch(memberOrderList));
+                }
                 row.close();
                 statement.close();
 

liquidnet-bus-service/liquidnet-service-platform/liquidnet-service-platform-impl/src/main/java/com/liquidnet/service/platform/service/impl/adam/dm/processor/DMRealNameProcessor.java

@@ -107,6 +107,10 @@ public class DMRealNameProcessor extends DataMigrationProcessorService {
                         realNameList.clear();
                     }
                 }
+                if (realNameList.size() > 0 && ct > 1000) {
+                    tl += realNameList.size();
+                    log.info("DM.execute.limit.result:{}", dmRealNameService.saveBatch(realNameList));
+                }
                 row.close();
                 statement.close();
 

liquidnet-bus-service/liquidnet-service-platform/liquidnet-service-platform-impl/src/main/java/com/liquidnet/service/platform/service/impl/adam/dm/processor/DMThirdPartsProcessor.java

@@ -116,6 +116,10 @@ public class DMThirdPartsProcessor extends DataMigrationProcessorService {
                         thirdPartyList.clear();
                     }
                 }
+                if (thirdPartyList.size() > 0 && ct > 1000) {
+                    tl += thirdPartyList.size();
+                    log.info("DM.execute.limit.result:{}", dmThirdPartyService.saveBatch(thirdPartyList));
+                }
                 row.close();
                 statement.close();
 

liquidnet-bus-service/liquidnet-service-platform/liquidnet-service-platform-impl/src/main/java/com/liquidnet/service/platform/service/impl/adam/dm/processor/DMTracesInfoProcessor.java

 package com.liquidnet.service.platform.service.impl.adam.dm.processor;
 
+import com.liquidnet.commons.lang.util.CollectionUtil;
 import com.liquidnet.service.adam.dto.vo.AdamMemberPriceVo;
 import com.liquidnet.service.adam.dto.vo.AdamMemberVo;
 import com.liquidnet.service.adam.entity.AdamMember;
@@ -141,7 +142,9 @@ public class DMTracesInfoProcessor extends DataMigrationProcessorService {
 
             AdamMemberPriceVo vo = AdamMemberPriceVo.getNew().copy(o);
 
-            dmRdmService.setMemberPriceVoByMemberId(vo.getMemberId(), Collections.singletonList(vo));
+            ArrayList<AdamMemberPriceVo> objectArrayList = new ArrayList<>();
+            objectArrayList.add(vo);
+            dmRdmService.setMemberPriceVoByMemberId(vo.getMemberId(), objectArrayList);
 
             memberPriceMapper.insert(o);
 //            mongoTemplate.insert(vo, AdamMemberPriceVo.class.getSimpleName());

liquidnet-bus-service/liquidnet-service-platform/liquidnet-service-platform-impl/src/main/java/com/liquidnet/service/platform/service/impl/adam/dm/processor/DMUserInformationProcessor.java

@@ -156,6 +156,10 @@ public class DMUserInformationProcessor extends DataMigrationProcessorService {
                         userInfoList.clear();
                     }
                 }
+                if (userList.size() > 0 && ct > 1000) {
+                    tl += userList.size();
+                    log.info("DM.execute.result:{} - {}", dmUserService.saveBatch(userList), dmUserInfoService.saveBatch(userInfoList));
+                }
                 row.close();
                 statement.close();
 

liquidnet-bus-service/liquidnet-service-platform/liquidnet-service-platform-impl/src/main/java/com/liquidnet/service/platform/service/impl/adam/dm/processor/DMUserMemberProcessor.java

@@ -107,6 +107,10 @@ public class DMUserMemberProcessor extends DataMigrationProcessorService {
                         userMemberList.clear();
                     }
                 }
+                if (userMemberList.size() > 0 && ct > 1000) {
+                    tl += userMemberList.size();
+                    log.info("DM.execute.limit.result:{}", dmUserMemberService.saveBatch(userMemberList));
+                }
                 row.close();
                 statement.close();
 

liquidnet-bus-service/liquidnet-service-platform/liquidnet-service-platform-impl/src/main/java/com/liquidnet/service/platform/utils/ObjectUtil.java

 package com.liquidnet.service.platform.utils;
 
-import com.liquidnet.service.adam.dto.vo.AdamEntersVo;
+import com.liquidnet.service.adam.dto.vo.*;
+import com.liquidnet.service.base.PagedResult;
 import com.liquidnet.service.kylin.dto.vo.returns.KylinOrderListVo;
 import com.mongodb.BasicDBObject;
 import org.springframework.stereotype.Component;
@@ -31,4 +32,67 @@ public class ObjectUtil {
     public static BasicDBObject cloneBasicDBObject() {
         return (BasicDBObject) basicDBObject.clone();
     }
+
+
+
+
+
+
+
+    private static final ArrayList<AdamTagParentVo> adamTagParentVoArrayList = new ArrayList<>();
+    private static final ArrayList<AdamTagVo> adamTagVoArrayList = new ArrayList<>();
+    private static final ArrayList<AdamMemberOrderSimpleVo> adamMemberOrderSimpleVoArrayList = new ArrayList<>();
+    private static final ArrayList<AdamThirdPartInfoVo> adamThirdPartInfoVoArrayList = new ArrayList<>();
+    private static final ArrayList<AdamEntersVo> adamEntersVoArrayList = new ArrayList<>();
+    private static final ArrayList<AdamAddressesVo> adamAddressesVoArrayList = new ArrayList<>();
+    private static final ArrayList<AdamMemberOrderVo> adamMemberOrderVoArrayList = new ArrayList<>();
+
+    private static final PagedResult<AdamCollectVo> adamCollectVoPagedResult = new PagedResult<>();
+    private static final PagedResult<AdamMemberOrderSimpleVo> adamMemberOrderSimpleVoPagedResult = new PagedResult<>();
+    private static final PagedResult<AdamDisposedVo> adamDisposedVoPagedResult = new PagedResult<>();
+    private static final PagedResult<AdamDisposedUserVo> adamDisposedUserVoPagedResult = new PagedResult<>();
+
+    public static ArrayList<AdamTagParentVo> getAdamTagParentVoArrayList() {
+        return (ArrayList<AdamTagParentVo>) adamTagParentVoArrayList.clone();
+    }
+
+    public static ArrayList<AdamTagVo> getAdamTagVoArrayList() {
+        return (ArrayList<AdamTagVo>) adamTagVoArrayList.clone();
+    }
+
+    public static ArrayList<AdamMemberOrderSimpleVo> getAdamMemberOrderSimpleVoArrayList() {
+        return (ArrayList<AdamMemberOrderSimpleVo>) adamMemberOrderSimpleVoArrayList.clone();
+    }
+
+    public static ArrayList<AdamThirdPartInfoVo> getAdamThirdPartInfoVoArrayList() {
+        return (ArrayList<AdamThirdPartInfoVo>) adamThirdPartInfoVoArrayList.clone();
+    }
+
+    public static ArrayList<AdamEntersVo> getAdamEntersVoArrayList() {
+        return (ArrayList<AdamEntersVo>) adamEntersVoArrayList.clone();
+    }
+
+    public static ArrayList<AdamAddressesVo> getAdamAddressesVoArrayList() {
+        return (ArrayList<AdamAddressesVo>) adamAddressesVoArrayList.clone();
+    }
+
+    public static ArrayList<AdamMemberOrderVo> getAdamMemberOrderVoArrayList() {
+        return (ArrayList<AdamMemberOrderVo>) adamMemberOrderVoArrayList.clone();
+    }
+
+    public static PagedResult<AdamCollectVo> getAdamCollectVoPagedResult() {
+        return adamCollectVoPagedResult.clone();
+    }
+
+    public static PagedResult<AdamMemberOrderSimpleVo> getAdamMemberOrderSimpleVoPagedResult() {
+        return adamMemberOrderSimpleVoPagedResult.clone();
+    }
+
+    public static PagedResult<AdamDisposedVo> getAdamDisposedVoPagedResult() {
+        return adamDisposedVoPagedResult.clone();
+    }
+
+    public static PagedResult<AdamDisposedUserVo> getAdamDisposedUserVoPagedResult() {
+        return adamDisposedUserVoPagedResult.clone();
+    }
 }