记得上下班打卡 | git大法好，push需谨慎

Skip to content

L
liquidnet-bus-v1
Commit 1ba51d62 authored by 张国柄's avatar 张国柄
Browse files
Options

dm;

parent ca01d2af
Show whitespace changes
Inline Side-by-side
Showing with 168 additions and 91 deletions
liquidnet-bus-api/liquidnet-service-adam-api/src/main/java/com/liquidnet/service/adam/dto/vo/AdamCollectVo.java
View file @ 1ba51d62
...@@ -2,6 +2,7 @@ package com.liquidnet.service.adam.dto.vo; ...@@ -2,6 +2,7 @@ package com.liquidnet.service.adam.dto.vo;
import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonFormat;
import com.liquidnet.commons.lang.util.DateUtil; import com.liquidnet.commons.lang.util.DateUtil;
import com.liquidnet.service.adam.entity.AdamCollection;
import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty; import io.swagger.annotations.ApiModelProperty;
import lombok.Data; import lombok.Data;
...@@ -31,4 +32,12 @@ public class AdamCollectVo implements Serializable, Cloneable { ...@@ -31,4 +32,12 @@ public class AdamCollectVo implements Serializable, Cloneable {
return new AdamCollectVo(); return new AdamCollectVo();
} }
} }
public AdamCollectVo copy(AdamCollection source) {
if (null == source) return this;
this.setContentId(source.getContentId());
this.setType(source.getType());
this.setCreatedAt(source.getCreatedAt());
return this;
}
} }
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMAddressesProcessor.java
View file @ 1ba51d62
...@@ -32,7 +32,6 @@ public class DMAddressesProcessor extends DataMigrationProcessorService { ...@@ -32,7 +32,6 @@ public class DMAddressesProcessor extends DataMigrationProcessorService {
String lS = request.getParameter(DM_LS), flg = StringUtils.defaultIfBlank(request.getParameter(DM_FLG), ""); String lS = request.getParameter(DM_LS), flg = StringUtils.defaultIfBlank(request.getParameter(DM_FLG), "");
if (StringUtils.isNotEmpty(request.getParameter(DM_FLUSH))) { if (StringUtils.isNotEmpty(request.getParameter(DM_FLUSH))) {
log.info("DM.flush.addresses:{}", userMapper.executeForDM("adam_addresses")); log.info("DM.flush.addresses:{}", userMapper.executeForDM("adam_addresses"));
log.info("DM.flush.AdamAddressesVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").exists(true)), AdamAddressesVo.class.getSimpleName()).getDeletedCount()); log.info("DM.flush.AdamAddressesVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").exists(true)), AdamAddressesVo.class.getSimpleName()).getDeletedCount());
} }
...@@ -40,19 +39,17 @@ public class DMAddressesProcessor extends DataMigrationProcessorService { ...@@ -40,19 +39,17 @@ public class DMAddressesProcessor extends DataMigrationProcessorService {
String field = "id,user_id,user_name,user_phone,province,city,county,user_address,defaultDelivery,created_at"; String field = "id,user_id,user_name,user_phone,province,city,county,user_address,defaultDelivery,created_at";
// 1-id,2-user_id,3-user_name,4-user_phone,5-province,6-city,7-county,8-user_address,9-defaultDelivery,10-created_at // 1-id,2-user_id,3-user_name,4-user_phone,5-province,6-city,7-county,8-user_address,9-defaultDelivery,10-created_at
String sql = sqlCount.replace("count(1)", field); String sql = sqlCount.replace("count(1)", field);
if ("1".equals(flg)) { if ("1".equals(flg)) {
sqlCount += " and user_id in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')"; sqlCount += " and user_id in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
sql += " and user_id in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')"; sql += " and user_id in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
} }
sql += " limit ?,?"; sql += " limit ?,?";
statement = connection.prepareStatement(sqlCount); statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
row.first(); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0; int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0;
log.info("DM.execute.counts:{}", ct); log.info("DM.execute.counts:{} ----------------------------> start", ct);
while (ct > 0) { while (ct > 0) {
statement = connection.prepareStatement(sql); statement = connection.prepareStatement(sql);
...@@ -60,7 +57,7 @@ public class DMAddressesProcessor extends DataMigrationProcessorService { ...@@ -60,7 +57,7 @@ public class DMAddressesProcessor extends DataMigrationProcessorService {
statement.setInt(1, ls); statement.setInt(1, ls);
statement.setInt(2, pSize); statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} ==> begin", ls, pSize); log.info("DM.execute.limit {},{} - begin", ls, pSize);
List<AdamAddresses> addressesList = new ArrayList<>(); List<AdamAddresses> addressesList = new ArrayList<>();
List<AdamAddressesVo> vos = new ArrayList<>(); List<AdamAddressesVo> vos = new ArrayList<>();
...@@ -83,20 +80,20 @@ public class DMAddressesProcessor extends DataMigrationProcessorService { ...@@ -83,20 +80,20 @@ public class DMAddressesProcessor extends DataMigrationProcessorService {
AdamAddressesVo vo = AdamAddressesVo.getNew().copy(addresses); AdamAddressesVo vo = AdamAddressesVo.getNew().copy(addresses);
// addressesMapper.insert(addresses);
// mongoTemplate.insert(vo, AdamAddressesVo.class.getSimpleName());
addressesList.add(addresses); addressesList.add(addresses);
vos.add(vo); vos.add(vo);
} }
} }
log.info("DM.execute.limit {},{} ==> handle.counts:{}", ls, pSize, addressesList.size()); log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, addressesList.size());
if (!CollectionUtils.isEmpty(addressesList)) { if (!CollectionUtils.isEmpty(addressesList)) {
tl += addressesList.size();
mongoTemplate.insert(vos, AdamAddressesVo.class.getSimpleName()); mongoTemplate.insert(vos, AdamAddressesVo.class.getSimpleName());
log.info("DM.execute.result:{}", adamAddressesService.saveBatch(addressesList)); log.info("DM.execute.limit.result:{}", adamAddressesService.saveBatch(addressesList));
} }
num++; num++;
ct -= pSize; ct -= pSize;
} }
log.info("DM.execute.counts:{} ~ {} ----------------------------> end", num * pSize, tl);
} }
} }
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMCollectionProcessor.java 0 → 100644
View file @ 1ba51d62
package com.liquidnet.service.adam.service.processor;
import com.liquidnet.commons.lang.util.ServletUtils;
import com.liquidnet.service.adam.dto.vo.AdamCollectVo;
import com.liquidnet.service.adam.entity.AdamCollection;
import com.liquidnet.service.adam.service.DataMigrationProcessorService;
import com.liquidnet.service.adam.service.IAdamCollectionService;
import com.mysql.cj.jdbc.result.ResultSetImpl;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import javax.servlet.http.HttpServletRequest;
import java.util.ArrayList;
import java.util.List;
@Slf4j
@Service
public class DMCollectionProcessor extends DataMigrationProcessorService {
@Autowired
IAdamCollectionService adamCollectionService;
@SneakyThrows
@Override
protected void dataProcessing() {
HttpServletRequest request = ServletUtils.getRequest();
String lS = request.getParameter(DM_LS), flg = StringUtils.defaultIfBlank(request.getParameter(DM_FLG), "");
if (StringUtils.isNotEmpty(request.getParameter(DM_FLUSH))) {
log.info("DM.flush.collection:{}", userMapper.executeForDM("adam_collection"));
log.info("DM.flush.AdamCollectVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").exists(true)), AdamCollectVo.class.getSimpleName()).getDeletedCount());
}
String sqlCount = "select count(1) from user_collections where type='TICKET' and status=1 and content_id>=5722";
String field = "uid,content_id,`type`,`status`,created_at,updated_at";
// 1-uid,2-content_id,3-`type`,4-`status`,5-created_at,6-updated_at
String sql = sqlCount.replace("count(1)", field);
if ("1".equals(flg)) {
sqlCount += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
sql += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
}
sql += " limit ?,?";
statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery();
row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0;
log.info("DM.execute.counts:{} ----------------------------> start", ct);
while (ct > 0) {
statement = connection.prepareStatement(sql);
int ls = (num == 0 && StringUtils.isNotEmpty(lS)) ? Integer.parseInt(lS) : pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} - begin", ls, pSize);
List<AdamCollection> collectionList = new ArrayList<>();
List<AdamCollectVo> vos = new ArrayList<>();
while (row.next()) {
String uid = row.getString(1);
if (!reUidList.contains(uid)) {
AdamCollection collection = new AdamCollection();
collection.setUid(uid);
collection.setContentId(row.getString(2));
collection.setType(row.getString(3));
collection.setState(row.getInt(4));
collection.setCreatedAt(row.getLocalDateTime(5));
collection.setUpdatedAt(row.getLocalDateTime(6));
collectionList.add(collection);
vos.add(AdamCollectVo.getNew().copy(collection));
}
}
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, collectionList.size());
if (!CollectionUtils.isEmpty(collectionList)) {
tl += collectionList.size();
mongoTemplate.insert(vos, AdamCollectVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamCollectionService.saveBatch(collectionList));
}
num++;
ct -= pSize;
}
log.info("DM.execute.counts:{} ~ {} ----------------------------> end", num * pSize, tl);
}
}
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMEntersProcessor.java
View file @ 1ba51d62
...@@ -40,20 +40,17 @@ public class DMEntersProcessor extends DataMigrationProcessorService { ...@@ -40,20 +40,17 @@ public class DMEntersProcessor extends DataMigrationProcessorService {
String field = "id,user_id,type,`name`,mobile,idcode,if(is_default='no', 0, 1),created_at"; String field = "id,user_id,type,`name`,mobile,idcode,if(is_default='no', 0, 1),created_at";
// 1-id,2-user_id,3-type,4-`name`,5-mobile,6-idcode,7-if(is_default='no', 0, 1),8-created_at // 1-id,2-user_id,3-type,4-`name`,5-mobile,6-idcode,7-if(is_default='no', 0, 1),8-created_at
String sql = sqlCount.replace("count(1)", field); String sql = sqlCount.replace("count(1)", field);
if ("1".equals(flg)) { if ("1".equals(flg)) {
sqlCount += " and user_id in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')"; sqlCount += " and user_id in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
sql += " and user_id in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')"; sql += " and user_id in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
} }
sql += " limit ?,?"; sql += " limit ?,?";
statement = connection.prepareStatement(sqlCount); statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
row.first(); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0; int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0;
log.info("DM.execute.counts:{}", ct); log.info("DM.execute.counts:{} ----------------------------> start", ct);
while (ct > 0) { while (ct > 0) {
statement = connection.prepareStatement(sql); statement = connection.prepareStatement(sql);
...@@ -61,7 +58,7 @@ public class DMEntersProcessor extends DataMigrationProcessorService { ...@@ -61,7 +58,7 @@ public class DMEntersProcessor extends DataMigrationProcessorService {
statement.setInt(1, ls); statement.setInt(1, ls);
statement.setInt(2, pSize); statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} ==> begin", ls, pSize); log.info("DM.execute.limit {},{} - begin", ls, pSize);
List<AdamEnters> entersList = new ArrayList<>(); List<AdamEnters> entersList = new ArrayList<>();
List<AdamEntersVo> vos = new ArrayList<>(); List<AdamEntersVo> vos = new ArrayList<>();
...@@ -82,20 +79,20 @@ public class DMEntersProcessor extends DataMigrationProcessorService { ...@@ -82,20 +79,20 @@ public class DMEntersProcessor extends DataMigrationProcessorService {
AdamEntersVo vo = AdamEntersVo.getNew().copy(enters); AdamEntersVo vo = AdamEntersVo.getNew().copy(enters);
// entersMapper.insert(enters);
// mongoTemplate.insert(vo, AdamEntersVo.class.getSimpleName());
entersList.add(enters); entersList.add(enters);
vos.add(vo); vos.add(vo);
} }
} }
log.info("DM.execute.limit {},{} ==> handle.counts:{}", ls, pSize, entersList.size()); log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, entersList.size());
if (!CollectionUtils.isEmpty(entersList)) { if (!CollectionUtils.isEmpty(entersList)) {
tl += entersList.size();
mongoTemplate.insert(vos, AdamEntersVo.class.getSimpleName()); mongoTemplate.insert(vos, AdamEntersVo.class.getSimpleName());
log.info("DM.execute.result:{}", adamEntersService.saveBatch(entersList)); log.info("DM.execute.limit.result:{}", adamEntersService.saveBatch(entersList));
} }
num++; num++;
ct -= pSize; ct -= pSize;
} }
log.info("DM.execute.counts:{} ~ {} ----------------------------> end", num * pSize, tl);
} }
} }
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMMemberCodeProcessor.java
View file @ 1ba51d62
...@@ -194,17 +194,13 @@ public class DMMemberCodeProcessor extends DataMigrationProcessorService { ...@@ -194,17 +194,13 @@ public class DMMemberCodeProcessor extends DataMigrationProcessorService {
// 1-vip_card_code,2-`type`+1,3-card_number,4-`status`,5-created_at,6-updated_at, // 1-vip_card_code,2-`type`+1,3-card_number,4-`status`,5-created_at,6-updated_at,
// 7-buy_order_id,8-buy_uid,9-buy_time,10-use_order_id,11-use_uid,12-use_time // 7-buy_order_id,8-buy_uid,9-buy_time,10-use_order_id,11-use_uid,12-use_time
String sql = sqlCount.replace("count(1)", field); String sql = sqlCount.replace("count(1)", field);
if ("1".equals(flg)) {
sqlCount += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
sql += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
}
sql += " limit ?,?"; sql += " limit ?,?";
statement = connection.prepareStatement(sqlCount); statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
row.first(); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0; int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0;
log.info("DM.execute.counts:{}", ct); log.info("DM.execute.counts:{} ----------------------------> start", ct);
while (ct > 0) { while (ct > 0) {
statement = connection.prepareStatement(sql); statement = connection.prepareStatement(sql);
...@@ -212,14 +208,14 @@ public class DMMemberCodeProcessor extends DataMigrationProcessorService { ...@@ -212,14 +208,14 @@ public class DMMemberCodeProcessor extends DataMigrationProcessorService {
statement.setInt(1, ls); statement.setInt(1, ls);
statement.setInt(2, pSize); statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} ==> begin", ls, pSize); log.info("DM.execute.limit {},{} - begin", ls, pSize);
List<AdamMemberCode> memberCodeList = new ArrayList<>(); List<AdamMemberCode> memberCodeList = new ArrayList<>();
List<AdamMemberCodeVo> vos = new ArrayList<>(); List<AdamMemberCodeVo> vos = new ArrayList<>();
while (row.next()) { while (row.next()) {
String buyUid = row.getString(10); // String buyUid = row.getString(8);
String useUid = row.getString(13); // String useUid = row.getString(11);
if (!reUidList.contains(buyUid) && reUidList.contains(useUid)) { // if (!reUidList.contains(buyUid) && reUidList.contains(useUid)) {
AdamMemberCode memberCode = new AdamMemberCode(); AdamMemberCode memberCode = new AdamMemberCode();
memberCode.setCode(row.getString(1)); memberCode.setCode(row.getString(1));
memberCode.setType(row.getInt(2)); memberCode.setType(row.getInt(2));
...@@ -238,16 +234,18 @@ public class DMMemberCodeProcessor extends DataMigrationProcessorService { ...@@ -238,16 +234,18 @@ public class DMMemberCodeProcessor extends DataMigrationProcessorService {
memberCodeList.add(memberCode); memberCodeList.add(memberCode);
vos.add(AdamMemberCodeVo.getNew().copy(memberCode)); vos.add(AdamMemberCodeVo.getNew().copy(memberCode));
// }
} }
} log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, memberCodeList.size());
log.info("DM.execute.limit {},{} ==> handle.counts:{}", ls, pSize, memberCodeList.size());
if (!CollectionUtils.isEmpty(memberCodeList)) { if (!CollectionUtils.isEmpty(memberCodeList)) {
tl += memberCodeList.size();
mongoTemplate.insert(vos, AdamMemberCodeVo.class.getSimpleName()); mongoTemplate.insert(vos, AdamMemberCodeVo.class.getSimpleName());
log.info("DM.execute.result:{}", adamMemberCodeService.saveBatch(memberCodeList)); log.info("DM.execute.limit.result:{}", adamMemberCodeService.saveBatch(memberCodeList));
} }
num++; num++;
ct -= pSize; ct -= pSize;
} }
log.info("DM.execute.counts:{} ~ {} ----------------------------> end", num * pSize, tl);
} }
} }
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMMemberOrderProcessor.java
View file @ 1ba51d62
...@@ -64,19 +64,17 @@ public class DMMemberOrderProcessor extends DataMigrationProcessorService { ...@@ -64,19 +64,17 @@ public class DMMemberOrderProcessor extends DataMigrationProcessorService {
// 1-order_code,2-uid,3-order_sub_type,4-vip_card_price,5-price,6-status,7-card_number,8-birthday, // 1-order_code,2-uid,3-order_sub_type,4-vip_card_price,5-price,6-status,7-card_number,8-birthday,
// 9-pay_type,10-pay_code,11-pay_at,12-created_at,13-updated_at,14-client_ip,15-source,16-version // 9-pay_type,10-pay_code,11-pay_at,12-created_at,13-updated_at,14-client_ip,15-source,16-version
String sql = sqlCount.replace("count(1)", field); String sql = sqlCount.replace("count(1)", field);
if ("1".equals(flg)) { if ("1".equals(flg)) {
sqlCount += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')"; sqlCount += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
sql += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')"; sql += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
} }
sql += " limit ?,?"; sql += " limit ?,?";
statement = connection.prepareStatement(sqlCount); statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
row.first(); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0; int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0;
log.info("DM.execute.counts:{}", ct); log.info("DM.execute.counts:{} ----------------------------> start", ct);
while (ct > 0) { while (ct > 0) {
...@@ -85,17 +83,16 @@ public class DMMemberOrderProcessor extends DataMigrationProcessorService { ...@@ -85,17 +83,16 @@ public class DMMemberOrderProcessor extends DataMigrationProcessorService {
statement.setInt(1, ls); statement.setInt(1, ls);
statement.setInt(2, pSize); statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} ==> begin", ls, pSize); log.info("DM.execute.limit {},{} - begin", ls, pSize);
List<AdamMemberOrder> memberOrderList = new ArrayList<>(); List<AdamMemberOrder> memberOrderList = new ArrayList<>();
List<AdamMemberOrderVo> vos = new ArrayList<>(); List<AdamMemberOrderVo> vos = new ArrayList<>();
while (row.next()) { while (row.next()) {
String uid = row.getString(1); String uid = row.getString(2);
if (!reUidList.contains(uid)) { if (!reUidList.contains(uid)) {
AdamMemberOrder memberOrder = new AdamMemberOrder(); AdamMemberOrder memberOrder = new AdamMemberOrder();
memberOrder.setOrderNo(row.getString(1)); memberOrder.setOrderNo(row.getString(1));
memberOrder.setUid(row.getString(2)); memberOrder.setUid(uid);
memberOrder.setMode(row.getInt(3)); memberOrder.setMode(row.getInt(3));
memberOrder.setPrice(row.getBigDecimal(4)); memberOrder.setPrice(row.getBigDecimal(4));
memberOrder.setPricePaid(row.getBigDecimal(5)); memberOrder.setPricePaid(row.getBigDecimal(5));
...@@ -119,20 +116,20 @@ public class DMMemberOrderProcessor extends DataMigrationProcessorService { ...@@ -119,20 +116,20 @@ public class DMMemberOrderProcessor extends DataMigrationProcessorService {
AdamMemberOrderVo vo = AdamMemberOrderVo.getNew().copy(memberOrder); AdamMemberOrderVo vo = AdamMemberOrderVo.getNew().copy(memberOrder);
// memberOrderMapper.insert(memberOrder);
// mongoTemplate.insert(vo, AdamMemberOrderVo.class.getSimpleName());
memberOrderList.add(memberOrder); memberOrderList.add(memberOrder);
vos.add(vo); vos.add(vo);
} }
} }
log.info("DM.execute.limit {},{} ==> handle.counts:{}", ls, pSize, memberOrderList.size()); log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, memberOrderList.size());
if (!CollectionUtils.isEmpty(memberOrderList)) { if (!CollectionUtils.isEmpty(memberOrderList)) {
tl += memberOrderList.size();
mongoTemplate.insert(vos, AdamMemberOrderVo.class.getSimpleName()); mongoTemplate.insert(vos, AdamMemberOrderVo.class.getSimpleName());
log.info("DM.execute.result:{}", adamMemberOrderService.saveBatch(memberOrderList)); log.info("DM.execute.limit.result:{}", adamMemberOrderService.saveBatch(memberOrderList));
} }
num++; num++;
ct -= pSize; ct -= pSize;
} }
log.info("DM.execute.counts:{} ~ {} ----------------------------> end", num * pSize, tl);
} }
} }
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMRealNameProcessor.java
View file @ 1ba51d62
...@@ -40,7 +40,6 @@ public class DMRealNameProcessor extends DataMigrationProcessorService { ...@@ -40,7 +40,6 @@ public class DMRealNameProcessor extends DataMigrationProcessorService {
String field = "id,uid,real_name,id_card,created_at,updated_at"; String field = "id,uid,real_name,id_card,created_at,updated_at";
// 1-id,2-uid,3-real_name,4-id_card,5-created_at,6-updated_at // 1-id,2-uid,3-real_name,4-id_card,5-created_at,6-updated_at
String sql = sqlCount.replace("count(1)", field); String sql = sqlCount.replace("count(1)", field);
if ("1".equals(flg)) { if ("1".equals(flg)) {
sqlCount += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')"; sqlCount += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
sql += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')"; sql += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
...@@ -50,8 +49,8 @@ public class DMRealNameProcessor extends DataMigrationProcessorService { ...@@ -50,8 +49,8 @@ public class DMRealNameProcessor extends DataMigrationProcessorService {
statement = connection.prepareStatement(sqlCount); statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
row.first(); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0; int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0;
log.info("DM.execute.counts:{}", ct); log.info("DM.execute.counts:{} ----------------------------> start", ct);
while (ct > 0) { while (ct > 0) {
statement = connection.prepareStatement(sql); statement = connection.prepareStatement(sql);
...@@ -59,13 +58,12 @@ public class DMRealNameProcessor extends DataMigrationProcessorService { ...@@ -59,13 +58,12 @@ public class DMRealNameProcessor extends DataMigrationProcessorService {
statement.setInt(1, ls); statement.setInt(1, ls);
statement.setInt(2, pSize); statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} ==> begin", ls, pSize); log.info("DM.execute.limit {},{} - begin", ls, pSize);
List<AdamRealName> realNameList = new ArrayList<>(); List<AdamRealName> realNameList = new ArrayList<>();
List<AdamRealInfoVo> vos = new ArrayList<>(); List<AdamRealInfoVo> vos = new ArrayList<>();
while (row.next()) { while (row.next()) {
String uid = row.getString(2); String uid = row.getString(2);
if (!reUidList.contains(uid)) { if (!reUidList.contains(uid)) {
AdamRealName realName = new AdamRealName(); AdamRealName realName = new AdamRealName();
realName.setRealNameId(row.getString(1)); realName.setRealNameId(row.getString(1));
...@@ -84,20 +82,20 @@ public class DMRealNameProcessor extends DataMigrationProcessorService { ...@@ -84,20 +82,20 @@ public class DMRealNameProcessor extends DataMigrationProcessorService {
vo.setIdCard(realName.getIdCard()); vo.setIdCard(realName.getIdCard());
vo.setState(1); vo.setState(1);
// realNameMapper.insert(realName);
// mongoTemplate.insert(vo, AdamRealInfoVo.class.getSimpleName());
realNameList.add(realName); realNameList.add(realName);
vos.add(vo); vos.add(vo);
} }
} }
log.info("DM.execute.limit {},{} ==> handle.counts:{}", ls, pSize, realNameList.size()); log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, realNameList.size());
if (!CollectionUtils.isEmpty(realNameList)) { if (!CollectionUtils.isEmpty(realNameList)) {
tl += realNameList.size();
mongoTemplate.insert(vos, AdamRealInfoVo.class.getSimpleName()); mongoTemplate.insert(vos, AdamRealInfoVo.class.getSimpleName());
log.info("DM.execute.result:{}", adamRealNameService.saveBatch(realNameList)); log.info("DM.execute.limit.result:{}", adamRealNameService.saveBatch(realNameList));
} }
num++; num++;
ct -= pSize; ct -= pSize;
} }
log.info("DM.execute.counts:{} ~ {} ----------------------------> end", num * pSize, tl);
} }
} }
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMThirdPartsProcessor.java
View file @ 1ba51d62
...@@ -40,20 +40,17 @@ public class DMThirdPartsProcessor extends DataMigrationProcessorService { ...@@ -40,20 +40,17 @@ public class DMThirdPartsProcessor extends DataMigrationProcessorService {
String field = "uid,open_id,avatar,nickname,type,created_at,updated_at"; String field = "uid,open_id,avatar,nickname,type,created_at,updated_at";
// 1-uid,2-open_id,3-avatar,4-nickname,5-type,6-created_at,7-updated_at // 1-uid,2-open_id,3-avatar,4-nickname,5-type,6-created_at,7-updated_at
String sql = sqlCount.replace("count(1)", field); String sql = sqlCount.replace("count(1)", field);
if ("1".equals(flg)) { if ("1".equals(flg)) {
sqlCount += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')"; sqlCount += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
sql += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')"; sql += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
} }
sql += " limit ?,?"; sql += " limit ?,?";
statement = connection.prepareStatement(sqlCount); statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
row.first(); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0; int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0;
log.info("DM.execute.counts:{}", ct); log.info("DM.execute.counts:{} ----------------------------> start", ct);
while (ct > 0) { while (ct > 0) {
statement = connection.prepareStatement(sql); statement = connection.prepareStatement(sql);
...@@ -61,13 +58,12 @@ public class DMThirdPartsProcessor extends DataMigrationProcessorService { ...@@ -61,13 +58,12 @@ public class DMThirdPartsProcessor extends DataMigrationProcessorService {
statement.setInt(1, ls); statement.setInt(1, ls);
statement.setInt(2, pSize); statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} ==> begin", ls, pSize); log.info("DM.execute.limit {},{} - begin", ls, pSize);
List<AdamThirdParty> thirdPartyList = new ArrayList<>(); List<AdamThirdParty> thirdPartyList = new ArrayList<>();
List<AdamThirdPartInfoVo> vos = new ArrayList<>(); List<AdamThirdPartInfoVo> vos = new ArrayList<>();
while (row.next()) { while (row.next()) {
String uid = row.getString(1); String uid = row.getString(1);
if (!reUidList.contains(uid)) { if (!reUidList.contains(uid)) {
AdamThirdParty thirdParty = new AdamThirdParty(); AdamThirdParty thirdParty = new AdamThirdParty();
thirdParty.setUid(uid); thirdParty.setUid(uid);
...@@ -89,20 +85,20 @@ public class DMThirdPartsProcessor extends DataMigrationProcessorService { ...@@ -89,20 +85,20 @@ public class DMThirdPartsProcessor extends DataMigrationProcessorService {
vo.setCreatedAt(thirdParty.getCreatedAt()); vo.setCreatedAt(thirdParty.getCreatedAt());
vo.setUpdatedAt(thirdParty.getUpdatedAt()); vo.setUpdatedAt(thirdParty.getUpdatedAt());
// thirdPartyMapper.insert(thirdParty);
// mongoTemplate.insert(vo, AdamThirdPartInfoVo.class.getSimpleName());
thirdPartyList.add(thirdParty); thirdPartyList.add(thirdParty);
vos.add(vo); vos.add(vo);
} }
} }
log.info("DM.execute.limit {},{} ==> handle.counts:{}", ls, pSize, thirdPartyList.size()); log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, thirdPartyList.size());
if (!CollectionUtils.isEmpty(thirdPartyList)) { if (!CollectionUtils.isEmpty(thirdPartyList)) {
tl += thirdPartyList.size();
mongoTemplate.insert(vos, AdamThirdPartInfoVo.class.getSimpleName()); mongoTemplate.insert(vos, AdamThirdPartInfoVo.class.getSimpleName());
log.info("DM.execute.result:{}", adamThirdPartyService.saveBatch(thirdPartyList)); log.info("DM.execute.limit.result:{}", adamThirdPartyService.saveBatch(thirdPartyList));
} }
num++; num++;
ct -= pSize; ct -= pSize;
} }
log.info("DM.execute.counts:{} ~ {} ----------------------------> end", num * pSize, tl);
} }
} }
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMUserInformationProcessor.java
View file @ 1ba51d62
...@@ -71,8 +71,8 @@ public class DMUserInformationProcessor extends DataMigrationProcessorService { ...@@ -71,8 +71,8 @@ public class DMUserInformationProcessor extends DataMigrationProcessorService {
statement = connection.prepareStatement(sqlCount); statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
row.first(); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0; int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0;
log.info("DM.execute.counts:{}", ct); log.info("DM.execute.counts:{} ----------------------------> start", ct);
while (ct > 0) { while (ct > 0) {
statement = connection.prepareStatement(sql); statement = connection.prepareStatement(sql);
...@@ -80,7 +80,7 @@ public class DMUserInformationProcessor extends DataMigrationProcessorService { ...@@ -80,7 +80,7 @@ public class DMUserInformationProcessor extends DataMigrationProcessorService {
statement.setInt(1, ls); statement.setInt(1, ls);
statement.setInt(2, pSize); statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} ==> begin", ls, pSize); log.info("DM.execute.limit {},{} - begin", ls, pSize);
List<AdamUser> userList = new ArrayList<>(); List<AdamUser> userList = new ArrayList<>();
List<AdamUserInfo> userInfoList = new ArrayList<>(); List<AdamUserInfo> userInfoList = new ArrayList<>();
...@@ -88,14 +88,11 @@ public class DMUserInformationProcessor extends DataMigrationProcessorService { ...@@ -88,14 +88,11 @@ public class DMUserInformationProcessor extends DataMigrationProcessorService {
while (row.next()) { while (row.next()) {
try { try {
AdamTagVo sex = sexMap.get(row.getString(4)); AdamTagVo sex = sexMap.get(row.getString(4));
String uid = row.getString(1); String uid = row.getString(1);
if (!reUidList.contains(uid)) { if (!reUidList.contains(uid)) {
// log.info("mobile:{}", user.getMobile()); // if (userMapper.selectCount(Wrappers.lambdaQuery(AdamUser.class).eq(AdamUser::getUid, uid)) > 0) {
if (userMapper.selectCount(Wrappers.lambdaQuery(AdamUser.class).eq(AdamUser::getUid, uid)) > 0) { // redisUtil.lSet(DM_EXISTS_USER, uid);
redisUtil.lSet(DM_EXISTS_USER, uid); // } else {
} else {
AdamUser user = new AdamUser(); AdamUser user = new AdamUser();
user.setUid(uid); user.setUid(uid);
user.setMobile(row.getString(2)); user.setMobile(row.getString(2));
...@@ -132,13 +129,10 @@ public class DMUserInformationProcessor extends DataMigrationProcessorService { ...@@ -132,13 +129,10 @@ public class DMUserInformationProcessor extends DataMigrationProcessorService {
vo.setIsComplete(row.getInt(12)); vo.setIsComplete(row.getInt(12));
vo.setQrCode(userInfo.getQrCode()); vo.setQrCode(userInfo.getQrCode());
// userMapper.insert(user);
// userInfoMapper.insert(userInfo);
// mongoTemplate.insert(vo, AdamUserInfoVo.class.getSimpleName());
userList.add(user); userList.add(user);
userInfoList.add(userInfo); userInfoList.add(userInfo);
vos.add(vo); vos.add(vo);
} // }
} }
} catch (SQLException throwables) { } catch (SQLException throwables) {
redisUtil.lSet(DM_FAILED_USER, row.getString(1)); redisUtil.lSet(DM_FAILED_USER, row.getString(1));
...@@ -146,15 +140,17 @@ public class DMUserInformationProcessor extends DataMigrationProcessorService { ...@@ -146,15 +140,17 @@ public class DMUserInformationProcessor extends DataMigrationProcessorService {
throwables.getErrorCode(), throwables.getMessage()); throwables.getErrorCode(), throwables.getMessage());
} }
} }
log.info("DM.execute.limit {},{} ==> handle.counts:{}", ls, pSize, userList.size()); log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, userList.size());
if (!CollectionUtils.isEmpty(userList)) { if (!CollectionUtils.isEmpty(userList)) {
tl += userList.size();
mongoTemplate.insert(vos, AdamUserInfoVo.class.getSimpleName()); mongoTemplate.insert(vos, AdamUserInfoVo.class.getSimpleName());
log.info("DM.execute.result:{}", adamUserService.saveBatch(userList)); log.info("DM.execute.limit.result:{}", adamUserService.saveBatch(userList));
log.info("DM.execute.result:{}", adamUserInfoService.saveBatch(userInfoList)); log.info("DM.execute.limit.result:{}", adamUserInfoService.saveBatch(userInfoList));
} }
num++; num++;
ct -= pSize; ct -= pSize;
} }
log.info("DM.execute.counts:{} ~ {} ----------------------------> end", num * pSize, tl);
} }
} }
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMUserMemberProcessor.java
View file @ 1ba51d62
...@@ -40,19 +40,17 @@ public class DMUserMemberProcessor extends DataMigrationProcessorService { ...@@ -40,19 +40,17 @@ public class DMUserMemberProcessor extends DataMigrationProcessorService {
String field = "uid,card_id,card_number,`status`,expired_at,created_at,updated_at"; String field = "uid,card_id,card_number,`status`,expired_at,created_at,updated_at";
// 1-uid,2-card_id,3-card_number,4-status,5-expired_at,6-created_at,7-updated_at // 1-uid,2-card_id,3-card_number,4-status,5-expired_at,6-created_at,7-updated_at
String sql = sqlCount.replace("count(1)", field); String sql = sqlCount.replace("count(1)", field);
if ("1".equals(flg)) { if ("1".equals(flg)) {
sqlCount += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')"; sqlCount += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
sql += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')"; sql += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
} }
sql += " limit ?,?"; sql += " limit ?,?";
statement = connection.prepareStatement(sqlCount); statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
row.first(); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0; int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0;
log.info("DM.execute.counts:{}", ct); log.info("DM.execute.counts:{} ----------------------------> start", ct);
while (ct > 0) { while (ct > 0) {
statement = connection.prepareStatement(sql); statement = connection.prepareStatement(sql);
...@@ -60,13 +58,12 @@ public class DMUserMemberProcessor extends DataMigrationProcessorService { ...@@ -60,13 +58,12 @@ public class DMUserMemberProcessor extends DataMigrationProcessorService {
statement.setInt(1, ls); statement.setInt(1, ls);
statement.setInt(2, pSize); statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} ==> begin", ls, pSize); log.info("DM.execute.limit {},{} - begin", ls, pSize);
List<AdamUserMember> userMemberList = new ArrayList<>(); List<AdamUserMember> userMemberList = new ArrayList<>();
List<AdamUserMemberVo> vos = new ArrayList<>(); List<AdamUserMemberVo> vos = new ArrayList<>();
while (row.next()) { while (row.next()) {
String uid = row.getString(1); String uid = row.getString(1);
if (!reUidList.contains(uid)) { if (!reUidList.contains(uid)) {
AdamUserMember userMember = new AdamUserMember(); AdamUserMember userMember = new AdamUserMember();
userMember.setUid(uid); userMember.setUid(uid);
...@@ -86,20 +83,20 @@ public class DMUserMemberProcessor extends DataMigrationProcessorService { ...@@ -86,20 +83,20 @@ public class DMUserMemberProcessor extends DataMigrationProcessorService {
vo.setCreatedAt(userMember.getCreatedAt()); vo.setCreatedAt(userMember.getCreatedAt());
vo.setUpdatedAt(userMember.getUpdatedAt()); vo.setUpdatedAt(userMember.getUpdatedAt());
// userMemberMapper.insert(userMember);
// mongoTemplate.insert(vo, AdamUserMemberVo.class.getSimpleName());
userMemberList.add(userMember); userMemberList.add(userMember);
vos.add(vo); vos.add(vo);
} }
} }
log.info("DM.execute.limit {},{} ==> handle.counts:{}", ls, pSize, userMemberList.size()); log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, userMemberList.size());
if (!CollectionUtils.isEmpty(userMemberList)) { if (!CollectionUtils.isEmpty(userMemberList)) {
tl += userMemberList.size();
mongoTemplate.insert(vos, AdamUserMemberVo.class.getSimpleName()); mongoTemplate.insert(vos, AdamUserMemberVo.class.getSimpleName());
log.info("DM.execute.result:{}", adamUserMemberService.saveBatch(userMemberList)); log.info("DM.execute.limit.result:{}", adamUserMemberService.saveBatch(userMemberList));
} }
num++; num++;
ct -= pSize; ct -= pSize;
} }
log.info("DM.execute.counts:{} ~ {} ----------------------------> end", num * pSize, tl);
} }
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment