记得上下班打卡 | git大法好，push需谨慎

Skip to content

L
liquidnet-bus-v1
Commit 1ba51d62 authored by 张国柄's avatar 张国柄
Browse files
Options

dm;

parent ca01d2af
Hide whitespace changes
Inline Side-by-side
Showing with 168 additions and 91 deletions
liquidnet-bus-api/liquidnet-service-adam-api/src/main/java/com/liquidnet/service/adam/dto/vo/AdamCollectVo.java
View file @ 1ba51d62
......@@ -2,6 +2,7 @@ package com.liquidnet.service.adam.dto.vo;
import com.fasterxml.jackson.annotation.JsonFormat;
import com.liquidnet.commons.lang.util.DateUtil;
import com.liquidnet.service.adam.entity.AdamCollection;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
......@@ -31,4 +32,12 @@ public class AdamCollectVo implements Serializable, Cloneable {
return new AdamCollectVo();
}
}
public AdamCollectVo copy(AdamCollection source) {
if (null == source) return this;
this.setContentId(source.getContentId());
this.setType(source.getType());
this.setCreatedAt(source.getCreatedAt());
return this;
}
}
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMAddressesProcessor.java
View file @ 1ba51d62
......@@ -32,7 +32,6 @@ public class DMAddressesProcessor extends DataMigrationProcessorService {
String lS = request.getParameter(DM_LS), flg = StringUtils.defaultIfBlank(request.getParameter(DM_FLG), "");
if (StringUtils.isNotEmpty(request.getParameter(DM_FLUSH))) {
log.info("DM.flush.addresses:{}", userMapper.executeForDM("adam_addresses"));
log.info("DM.flush.AdamAddressesVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").exists(true)), AdamAddressesVo.class.getSimpleName()).getDeletedCount());
}
......@@ -40,19 +39,17 @@ public class DMAddressesProcessor extends DataMigrationProcessorService {
String field = "id,user_id,user_name,user_phone,province,city,county,user_address,defaultDelivery,created_at";
// 1-id,2-user_id,3-user_name,4-user_phone,5-province,6-city,7-county,8-user_address,9-defaultDelivery,10-created_at
String sql = sqlCount.replace("count(1)", field);
if ("1".equals(flg)) {
sqlCount += " and user_id in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
sql += " and user_id in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
}
sql += " limit ?,?";
statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery();
row.first();
int ct = row.getInt(1), pSize = 1000, num = 0;
log.info("DM.execute.counts:{}", ct);
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0;
log.info("DM.execute.counts:{} ----------------------------> start", ct);
while (ct > 0) {
statement = connection.prepareStatement(sql);
......@@ -60,7 +57,7 @@ public class DMAddressesProcessor extends DataMigrationProcessorService {
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} ==> begin", ls, pSize);
log.info("DM.execute.limit {},{} - begin", ls, pSize);
List<AdamAddresses> addressesList = new ArrayList<>();
List<AdamAddressesVo> vos = new ArrayList<>();
......@@ -83,20 +80,20 @@ public class DMAddressesProcessor extends DataMigrationProcessorService {
AdamAddressesVo vo = AdamAddressesVo.getNew().copy(addresses);
// addressesMapper.insert(addresses);
// mongoTemplate.insert(vo, AdamAddressesVo.class.getSimpleName());
addressesList.add(addresses);
vos.add(vo);
}
}
log.info("DM.execute.limit {},{} ==> handle.counts:{}", ls, pSize, addressesList.size());
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, addressesList.size());
if (!CollectionUtils.isEmpty(addressesList)) {
tl += addressesList.size();
mongoTemplate.insert(vos, AdamAddressesVo.class.getSimpleName());
log.info("DM.execute.result:{}", adamAddressesService.saveBatch(addressesList));
log.info("DM.execute.limit.result:{}", adamAddressesService.saveBatch(addressesList));
}
num++;
ct -= pSize;
}
log.info("DM.execute.counts:{} ~ {} ----------------------------> end", num * pSize, tl);
}
}
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMCollectionProcessor.java 0 → 100644
View file @ 1ba51d62
package com.liquidnet.service.adam.service.processor;
import com.liquidnet.commons.lang.util.ServletUtils;
import com.liquidnet.service.adam.dto.vo.AdamCollectVo;
import com.liquidnet.service.adam.entity.AdamCollection;
import com.liquidnet.service.adam.service.DataMigrationProcessorService;
import com.liquidnet.service.adam.service.IAdamCollectionService;
import com.mysql.cj.jdbc.result.ResultSetImpl;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import javax.servlet.http.HttpServletRequest;
import java.util.ArrayList;
import java.util.List;
@Slf4j
@Service
public class DMCollectionProcessor extends DataMigrationProcessorService {
@Autowired
IAdamCollectionService adamCollectionService;
@SneakyThrows
@Override
protected void dataProcessing() {
HttpServletRequest request = ServletUtils.getRequest();
String lS = request.getParameter(DM_LS), flg = StringUtils.defaultIfBlank(request.getParameter(DM_FLG), "");
if (StringUtils.isNotEmpty(request.getParameter(DM_FLUSH))) {
log.info("DM.flush.collection:{}", userMapper.executeForDM("adam_collection"));
log.info("DM.flush.AdamCollectVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").exists(true)), AdamCollectVo.class.getSimpleName()).getDeletedCount());
}
String sqlCount = "select count(1) from user_collections where type='TICKET' and status=1 and content_id>=5722";
String field = "uid,content_id,`type`,`status`,created_at,updated_at";
// 1-uid,2-content_id,3-`type`,4-`status`,5-created_at,6-updated_at
String sql = sqlCount.replace("count(1)", field);
if ("1".equals(flg)) {
sqlCount += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
sql += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
}
sql += " limit ?,?";
statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery();
row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0;
log.info("DM.execute.counts:{} ----------------------------> start", ct);
while (ct > 0) {
statement = connection.prepareStatement(sql);
int ls = (num == 0 && StringUtils.isNotEmpty(lS)) ? Integer.parseInt(lS) : pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} - begin", ls, pSize);
List<AdamCollection> collectionList = new ArrayList<>();
List<AdamCollectVo> vos = new ArrayList<>();
while (row.next()) {
String uid = row.getString(1);
if (!reUidList.contains(uid)) {
AdamCollection collection = new AdamCollection();
collection.setUid(uid);
collection.setContentId(row.getString(2));
collection.setType(row.getString(3));
collection.setState(row.getInt(4));
collection.setCreatedAt(row.getLocalDateTime(5));
collection.setUpdatedAt(row.getLocalDateTime(6));
collectionList.add(collection);
vos.add(AdamCollectVo.getNew().copy(collection));
}
}
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, collectionList.size());
if (!CollectionUtils.isEmpty(collectionList)) {
tl += collectionList.size();
mongoTemplate.insert(vos, AdamCollectVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamCollectionService.saveBatch(collectionList));
}
num++;
ct -= pSize;
}
log.info("DM.execute.counts:{} ~ {} ----------------------------> end", num * pSize, tl);
}
}
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMEntersProcessor.java
View file @ 1ba51d62
......@@ -40,20 +40,17 @@ public class DMEntersProcessor extends DataMigrationProcessorService {
String field = "id,user_id,type,`name`,mobile,idcode,if(is_default='no', 0, 1),created_at";
// 1-id,2-user_id,3-type,4-`name`,5-mobile,6-idcode,7-if(is_default='no', 0, 1),8-created_at
String sql = sqlCount.replace("count(1)", field);
if ("1".equals(flg)) {
sqlCount += " and user_id in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
sql += " and user_id in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
}
sql += " limit ?,?";
statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery();
row.first();
int ct = row.getInt(1), pSize = 1000, num = 0;
log.info("DM.execute.counts:{}", ct);
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0;
log.info("DM.execute.counts:{} ----------------------------> start", ct);
while (ct > 0) {
statement = connection.prepareStatement(sql);
......@@ -61,7 +58,7 @@ public class DMEntersProcessor extends DataMigrationProcessorService {
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} ==> begin", ls, pSize);
log.info("DM.execute.limit {},{} - begin", ls, pSize);
List<AdamEnters> entersList = new ArrayList<>();
List<AdamEntersVo> vos = new ArrayList<>();
......@@ -82,20 +79,20 @@ public class DMEntersProcessor extends DataMigrationProcessorService {
AdamEntersVo vo = AdamEntersVo.getNew().copy(enters);
// entersMapper.insert(enters);
// mongoTemplate.insert(vo, AdamEntersVo.class.getSimpleName());
entersList.add(enters);
vos.add(vo);
}
}
log.info("DM.execute.limit {},{} ==> handle.counts:{}", ls, pSize, entersList.size());
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, entersList.size());
if (!CollectionUtils.isEmpty(entersList)) {
tl += entersList.size();
mongoTemplate.insert(vos, AdamEntersVo.class.getSimpleName());
log.info("DM.execute.result:{}", adamEntersService.saveBatch(entersList));
log.info("DM.execute.limit.result:{}", adamEntersService.saveBatch(entersList));
}
num++;
ct -= pSize;
}
log.info("DM.execute.counts:{} ~ {} ----------------------------> end", num * pSize, tl);
}
}
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMMemberCodeProcessor.java
View file @ 1ba51d62
......@@ -194,17 +194,13 @@ public class DMMemberCodeProcessor extends DataMigrationProcessorService {
// 1-vip_card_code,2-`type`+1,3-card_number,4-`status`,5-created_at,6-updated_at,
// 7-buy_order_id,8-buy_uid,9-buy_time,10-use_order_id,11-use_uid,12-use_time
String sql = sqlCount.replace("count(1)", field);
if ("1".equals(flg)) {
sqlCount += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
sql += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
}
sql += " limit ?,?";
statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery();
row.first();
int ct = row.getInt(1), pSize = 1000, num = 0;
log.info("DM.execute.counts:{}", ct);
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0;
log.info("DM.execute.counts:{} ----------------------------> start", ct);
while (ct > 0) {
statement = connection.prepareStatement(sql);
......@@ -212,14 +208,14 @@ public class DMMemberCodeProcessor extends DataMigrationProcessorService {
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} ==> begin", ls, pSize);
log.info("DM.execute.limit {},{} - begin", ls, pSize);
List<AdamMemberCode> memberCodeList = new ArrayList<>();
List<AdamMemberCodeVo> vos = new ArrayList<>();
while (row.next()) {
String buyUid = row.getString(10);
String useUid = row.getString(13);
if (!reUidList.contains(buyUid) && reUidList.contains(useUid)) {
// String buyUid = row.getString(8);
// String useUid = row.getString(11);
// if (!reUidList.contains(buyUid) && reUidList.contains(useUid)) {
AdamMemberCode memberCode = new AdamMemberCode();
memberCode.setCode(row.getString(1));
memberCode.setType(row.getInt(2));
......@@ -238,16 +234,18 @@ public class DMMemberCodeProcessor extends DataMigrationProcessorService {
memberCodeList.add(memberCode);
vos.add(AdamMemberCodeVo.getNew().copy(memberCode));
}
// }
}
log.info("DM.execute.limit {},{} ==> handle.counts:{}", ls, pSize, memberCodeList.size());
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, memberCodeList.size());
if (!CollectionUtils.isEmpty(memberCodeList)) {
tl += memberCodeList.size();
mongoTemplate.insert(vos, AdamMemberCodeVo.class.getSimpleName());
log.info("DM.execute.result:{}", adamMemberCodeService.saveBatch(memberCodeList));
log.info("DM.execute.limit.result:{}", adamMemberCodeService.saveBatch(memberCodeList));
}
num++;
ct -= pSize;
}
log.info("DM.execute.counts:{} ~ {} ----------------------------> end", num * pSize, tl);
}
}
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMMemberOrderProcessor.java
View file @ 1ba51d62
......@@ -64,19 +64,17 @@ public class DMMemberOrderProcessor extends DataMigrationProcessorService {
// 1-order_code,2-uid,3-order_sub_type,4-vip_card_price,5-price,6-status,7-card_number,8-birthday,
// 9-pay_type,10-pay_code,11-pay_at,12-created_at,13-updated_at,14-client_ip,15-source,16-version
String sql = sqlCount.replace("count(1)", field);
if ("1".equals(flg)) {
sqlCount += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
sql += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
}
sql += " limit ?,?";
statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery();
row.first();
int ct = row.getInt(1), pSize = 1000, num = 0;
log.info("DM.execute.counts:{}", ct);
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0;
log.info("DM.execute.counts:{} ----------------------------> start", ct);
while (ct > 0) {
......@@ -85,17 +83,16 @@ public class DMMemberOrderProcessor extends DataMigrationProcessorService {
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} ==> begin", ls, pSize);
log.info("DM.execute.limit {},{} - begin", ls, pSize);
List<AdamMemberOrder> memberOrderList = new ArrayList<>();
List<AdamMemberOrderVo> vos = new ArrayList<>();
while (row.next()) {
String uid = row.getString(1);
String uid = row.getString(2);
if (!reUidList.contains(uid)) {
AdamMemberOrder memberOrder = new AdamMemberOrder();
memberOrder.setOrderNo(row.getString(1));
memberOrder.setUid(row.getString(2));
memberOrder.setUid(uid);
memberOrder.setMode(row.getInt(3));
memberOrder.setPrice(row.getBigDecimal(4));
memberOrder.setPricePaid(row.getBigDecimal(5));
......@@ -119,20 +116,20 @@ public class DMMemberOrderProcessor extends DataMigrationProcessorService {
AdamMemberOrderVo vo = AdamMemberOrderVo.getNew().copy(memberOrder);
// memberOrderMapper.insert(memberOrder);
// mongoTemplate.insert(vo, AdamMemberOrderVo.class.getSimpleName());
memberOrderList.add(memberOrder);
vos.add(vo);
}
}
log.info("DM.execute.limit {},{} ==> handle.counts:{}", ls, pSize, memberOrderList.size());
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, memberOrderList.size());
if (!CollectionUtils.isEmpty(memberOrderList)) {
tl += memberOrderList.size();
mongoTemplate.insert(vos, AdamMemberOrderVo.class.getSimpleName());
log.info("DM.execute.result:{}", adamMemberOrderService.saveBatch(memberOrderList));
log.info("DM.execute.limit.result:{}", adamMemberOrderService.saveBatch(memberOrderList));
}
num++;
ct -= pSize;
}
log.info("DM.execute.counts:{} ~ {} ----------------------------> end", num * pSize, tl);
}
}
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMRealNameProcessor.java
View file @ 1ba51d62
......@@ -40,7 +40,6 @@ public class DMRealNameProcessor extends DataMigrationProcessorService {
String field = "id,uid,real_name,id_card,created_at,updated_at";
// 1-id,2-uid,3-real_name,4-id_card,5-created_at,6-updated_at
String sql = sqlCount.replace("count(1)", field);
if ("1".equals(flg)) {
sqlCount += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
sql += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
......@@ -50,8 +49,8 @@ public class DMRealNameProcessor extends DataMigrationProcessorService {
statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery();
row.first();
int ct = row.getInt(1), pSize = 1000, num = 0;
log.info("DM.execute.counts:{}", ct);
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0;
log.info("DM.execute.counts:{} ----------------------------> start", ct);
while (ct > 0) {
statement = connection.prepareStatement(sql);
......@@ -59,13 +58,12 @@ public class DMRealNameProcessor extends DataMigrationProcessorService {
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} ==> begin", ls, pSize);
log.info("DM.execute.limit {},{} - begin", ls, pSize);
List<AdamRealName> realNameList = new ArrayList<>();
List<AdamRealInfoVo> vos = new ArrayList<>();
while (row.next()) {
String uid = row.getString(2);
if (!reUidList.contains(uid)) {
AdamRealName realName = new AdamRealName();
realName.setRealNameId(row.getString(1));
......@@ -84,20 +82,20 @@ public class DMRealNameProcessor extends DataMigrationProcessorService {
vo.setIdCard(realName.getIdCard());
vo.setState(1);
// realNameMapper.insert(realName);
// mongoTemplate.insert(vo, AdamRealInfoVo.class.getSimpleName());
realNameList.add(realName);
vos.add(vo);
}
}
log.info("DM.execute.limit {},{} ==> handle.counts:{}", ls, pSize, realNameList.size());
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, realNameList.size());
if (!CollectionUtils.isEmpty(realNameList)) {
tl += realNameList.size();
mongoTemplate.insert(vos, AdamRealInfoVo.class.getSimpleName());
log.info("DM.execute.result:{}", adamRealNameService.saveBatch(realNameList));
log.info("DM.execute.limit.result:{}", adamRealNameService.saveBatch(realNameList));
}
num++;
ct -= pSize;
}
log.info("DM.execute.counts:{} ~ {} ----------------------------> end", num * pSize, tl);
}
}
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMThirdPartsProcessor.java
View file @ 1ba51d62
......@@ -40,20 +40,17 @@ public class DMThirdPartsProcessor extends DataMigrationProcessorService {
String field = "uid,open_id,avatar,nickname,type,created_at,updated_at";
// 1-uid,2-open_id,3-avatar,4-nickname,5-type,6-created_at,7-updated_at
String sql = sqlCount.replace("count(1)", field);
if ("1".equals(flg)) {
sqlCount += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
sql += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
}
sql += " limit ?,?";
statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery();
row.first();
int ct = row.getInt(1), pSize = 1000, num = 0;
log.info("DM.execute.counts:{}", ct);
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0;
log.info("DM.execute.counts:{} ----------------------------> start", ct);
while (ct > 0) {
statement = connection.prepareStatement(sql);
......@@ -61,13 +58,12 @@ public class DMThirdPartsProcessor extends DataMigrationProcessorService {
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} ==> begin", ls, pSize);
log.info("DM.execute.limit {},{} - begin", ls, pSize);
List<AdamThirdParty> thirdPartyList = new ArrayList<>();
List<AdamThirdPartInfoVo> vos = new ArrayList<>();
while (row.next()) {
String uid = row.getString(1);
if (!reUidList.contains(uid)) {
AdamThirdParty thirdParty = new AdamThirdParty();
thirdParty.setUid(uid);
......@@ -89,20 +85,20 @@ public class DMThirdPartsProcessor extends DataMigrationProcessorService {
vo.setCreatedAt(thirdParty.getCreatedAt());
vo.setUpdatedAt(thirdParty.getUpdatedAt());
// thirdPartyMapper.insert(thirdParty);
// mongoTemplate.insert(vo, AdamThirdPartInfoVo.class.getSimpleName());
thirdPartyList.add(thirdParty);
vos.add(vo);
}
}
log.info("DM.execute.limit {},{} ==> handle.counts:{}", ls, pSize, thirdPartyList.size());
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, thirdPartyList.size());
if (!CollectionUtils.isEmpty(thirdPartyList)) {
tl += thirdPartyList.size();
mongoTemplate.insert(vos, AdamThirdPartInfoVo.class.getSimpleName());
log.info("DM.execute.result:{}", adamThirdPartyService.saveBatch(thirdPartyList));
log.info("DM.execute.limit.result:{}", adamThirdPartyService.saveBatch(thirdPartyList));
}
num++;
ct -= pSize;
}
log.info("DM.execute.counts:{} ~ {} ----------------------------> end", num * pSize, tl);
}
}
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMUserInformationProcessor.java
View file @ 1ba51d62
......@@ -71,8 +71,8 @@ public class DMUserInformationProcessor extends DataMigrationProcessorService {
statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery();
row.first();
int ct = row.getInt(1), pSize = 1000, num = 0;
log.info("DM.execute.counts:{}", ct);
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0;
log.info("DM.execute.counts:{} ----------------------------> start", ct);
while (ct > 0) {
statement = connection.prepareStatement(sql);
......@@ -80,7 +80,7 @@ public class DMUserInformationProcessor extends DataMigrationProcessorService {
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} ==> begin", ls, pSize);
log.info("DM.execute.limit {},{} - begin", ls, pSize);
List<AdamUser> userList = new ArrayList<>();
List<AdamUserInfo> userInfoList = new ArrayList<>();
......@@ -88,14 +88,11 @@ public class DMUserInformationProcessor extends DataMigrationProcessorService {
while (row.next()) {
try {
AdamTagVo sex = sexMap.get(row.getString(4));
String uid = row.getString(1);
if (!reUidList.contains(uid)) {
// log.info("mobile:{}", user.getMobile());
if (userMapper.selectCount(Wrappers.lambdaQuery(AdamUser.class).eq(AdamUser::getUid, uid)) > 0) {
redisUtil.lSet(DM_EXISTS_USER, uid);
} else {
// if (userMapper.selectCount(Wrappers.lambdaQuery(AdamUser.class).eq(AdamUser::getUid, uid)) > 0) {
// redisUtil.lSet(DM_EXISTS_USER, uid);
// } else {
AdamUser user = new AdamUser();
user.setUid(uid);
user.setMobile(row.getString(2));
......@@ -132,13 +129,10 @@ public class DMUserInformationProcessor extends DataMigrationProcessorService {
vo.setIsComplete(row.getInt(12));
vo.setQrCode(userInfo.getQrCode());
// userMapper.insert(user);
// userInfoMapper.insert(userInfo);
// mongoTemplate.insert(vo, AdamUserInfoVo.class.getSimpleName());
userList.add(user);
userInfoList.add(userInfo);
vos.add(vo);
}
// }
}
} catch (SQLException throwables) {
redisUtil.lSet(DM_FAILED_USER, row.getString(1));
......@@ -146,15 +140,17 @@ public class DMUserInformationProcessor extends DataMigrationProcessorService {
throwables.getErrorCode(), throwables.getMessage());
}
}
log.info("DM.execute.limit {},{} ==> handle.counts:{}", ls, pSize, userList.size());
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, userList.size());
if (!CollectionUtils.isEmpty(userList)) {
tl += userList.size();
mongoTemplate.insert(vos, AdamUserInfoVo.class.getSimpleName());
log.info("DM.execute.result:{}", adamUserService.saveBatch(userList));
log.info("DM.execute.result:{}", adamUserInfoService.saveBatch(userInfoList));
log.info("DM.execute.limit.result:{}", adamUserService.saveBatch(userList));
log.info("DM.execute.limit.result:{}", adamUserInfoService.saveBatch(userInfoList));
}
num++;
ct -= pSize;
}
log.info("DM.execute.counts:{} ~ {} ----------------------------> end", num * pSize, tl);
}
}
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMUserMemberProcessor.java
View file @ 1ba51d62
......@@ -40,19 +40,17 @@ public class DMUserMemberProcessor extends DataMigrationProcessorService {
String field = "uid,card_id,card_number,`status`,expired_at,created_at,updated_at";
// 1-uid,2-card_id,3-card_number,4-status,5-expired_at,6-created_at,7-updated_at
String sql = sqlCount.replace("count(1)", field);
if ("1".equals(flg)) {
sqlCount += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
sql += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
}
sql += " limit ?,?";
statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery();
row.first();
int ct = row.getInt(1), pSize = 1000, num = 0;
log.info("DM.execute.counts:{}", ct);
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0;
log.info("DM.execute.counts:{} ----------------------------> start", ct);
while (ct > 0) {
statement = connection.prepareStatement(sql);
......@@ -60,13 +58,12 @@ public class DMUserMemberProcessor extends DataMigrationProcessorService {
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} ==> begin", ls, pSize);
log.info("DM.execute.limit {},{} - begin", ls, pSize);
List<AdamUserMember> userMemberList = new ArrayList<>();
List<AdamUserMemberVo> vos = new ArrayList<>();
while (row.next()) {
String uid = row.getString(1);
if (!reUidList.contains(uid)) {
AdamUserMember userMember = new AdamUserMember();
userMember.setUid(uid);
......@@ -86,20 +83,20 @@ public class DMUserMemberProcessor extends DataMigrationProcessorService {
vo.setCreatedAt(userMember.getCreatedAt());
vo.setUpdatedAt(userMember.getUpdatedAt());
// userMemberMapper.insert(userMember);
// mongoTemplate.insert(vo, AdamUserMemberVo.class.getSimpleName());
userMemberList.add(userMember);
vos.add(vo);
}
}
log.info("DM.execute.limit {},{} ==> handle.counts:{}", ls, pSize, userMemberList.size());
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, userMemberList.size());
if (!CollectionUtils.isEmpty(userMemberList)) {
tl += userMemberList.size();
mongoTemplate.insert(vos, AdamUserMemberVo.class.getSimpleName());
log.info("DM.execute.result:{}", adamUserMemberService.saveBatch(userMemberList));
log.info("DM.execute.limit.result:{}", adamUserMemberService.saveBatch(userMemberList));
}
num++;
ct -= pSize;
}
log.info("DM.execute.counts:{} ~ {} ----------------------------> end", num * pSize, tl);
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment