记得上下班打卡 | git大法好，push需谨慎

Skip to content

L
liquidnet-bus-v1
Commit 5f5b421b authored by 张国柄's avatar 张国柄
Browse files
Options

dm;

parent 2684c9b9
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 248 additions and 125 deletions
liquidnet-bus-api/liquidnet-service-adam-api/src/main/java/com/liquidnet/service/adam/service/IAdamMemberCodeService.java 0 → 100644
View file @ 5f5b421b
package com.liquidnet.service.adam.service;
import com.liquidnet.service.adam.entity.AdamMemberCode;
import com.baomidou.mybatisplus.extension.service.IService;
/**
* <p>
* 服务类
* </p>
*
* @author liquidnet
* @since 2021-06-23
*/
public interface IAdamMemberCodeService extends IService<AdamMemberCode> {
}
liquidnet-bus-api/liquidnet-service-adam-api/src/main/java/com/liquidnet/service/adam/service/IAdamThirdPartyService.java 0 → 100644
View file @ 5f5b421b
package com.liquidnet.service.adam.service;
import com.liquidnet.service.adam.entity.AdamThirdParty;
import com.baomidou.mybatisplus.extension.service.IService;
/**
* <p>
* 用户第三方账号信息 服务类
* </p>
*
* @author liquidnet
* @since 2021-06-23
*/
public interface IAdamThirdPartyService extends IService<AdamThirdParty> {
}
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/impl/AdamMemberCodeServiceImpl.java 0 → 100644
View file @ 5f5b421b
package com.liquidnet.service.adam.service.impl;
import com.liquidnet.service.adam.entity.AdamMemberCode;
import com.liquidnet.service.adam.mapper.AdamMemberCodeMapper;
import com.liquidnet.service.adam.service.IAdamMemberCodeService;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import org.springframework.stereotype.Service;
/**
* <p>
* 服务实现类
* </p>
*
* @author liquidnet
* @since 2021-06-23
*/
@Service
public class AdamMemberCodeServiceImpl extends ServiceImpl<AdamMemberCodeMapper, AdamMemberCode> implements IAdamMemberCodeService {
}
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/impl/AdamThirdPartyServiceImpl.java 0 → 100644
View file @ 5f5b421b
package com.liquidnet.service.adam.service.impl;
import com.liquidnet.service.adam.entity.AdamThirdParty;
import com.liquidnet.service.adam.mapper.AdamThirdPartyMapper;
import com.liquidnet.service.adam.service.IAdamThirdPartyService;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import org.springframework.stereotype.Service;
/**
* <p>
* 用户第三方账号信息 服务实现类
* </p>
*
* @author liquidnet
* @since 2021-06-23
*/
@Service
public class AdamThirdPartyServiceImpl extends ServiceImpl<AdamThirdPartyMapper, AdamThirdParty> implements IAdamThirdPartyService {
}
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMAddressesProcessor.java
View file @ 5f5b421b
package com.liquidnet.service.adam.service.processor;
import com.liquidnet.common.cache.redis.util.RedisUtil;
import com.liquidnet.commons.lang.util.ServletUtils;
import com.liquidnet.service.adam.dto.vo.AdamAddressesVo;
import com.liquidnet.service.adam.dto.vo.AdamUserInfoVo;
import com.liquidnet.service.adam.entity.AdamAddresses;
import com.liquidnet.service.adam.mapper.AdamAddressesMapper;
import com.liquidnet.service.adam.service.DataMigrationProcessorService;
import com.liquidnet.service.adam.service.IAdamAddressesService;
import com.mysql.cj.jdbc.result.ResultSetImpl;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import javax.servlet.http.HttpServletRequest;
import java.util.ArrayList;
import java.util.List;
@Slf4j
@Service
public class DMAddressesProcessor extends DataMigrationProcessorService {
@Autowired
MongoTemplate mongoTemplate;
@Autowired
RedisUtil redisUtil;
@Autowired
AdamAddressesMapper addressesMapper;
IAdamAddressesService adamAddressesService;
@SneakyThrows
@Override
protected void dataProcessing() {
HttpServletRequest request = ServletUtils.getRequest();
String flg = StringUtils.defaultIfBlank(request.getParameter(DM_FLG), "");
String lS = request.getParameter(DM_LS), flg = StringUtils.defaultIfBlank(request.getParameter(DM_FLG), "");
if (StringUtils.isNotEmpty(request.getParameter(DM_FLUSH))) {
log.info("DM.flush.addresses:{}", userMapper.executeForDM("adam_addresses"));
log.info("DM.flush.AdamAddressesVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").exists(true)), AdamAddressesVo.class.getSimpleName()).getDeletedCount());
log.info("DM.flush.AdamAddressesVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").ne(null)), AdamAddressesVo.class.getSimpleName()).getDeletedCount());
}
String sqlCount = "select count(1) from addresses where 1=1";
......@@ -55,16 +51,19 @@ public class DMAddressesProcessor extends DataMigrationProcessorService {
statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery();
row.first();
int ct = row.getInt(1), pSize = 2, num = 0;
log.info("DM数据迁移总数:{}", ct);
int ct = row.getInt(1), pSize = 1000, num = 0;
log.info("DM.execute.counts:{}", ct);
while (ct > 0) {
statement = connection.prepareStatement(sql);
statement.setInt(1, pSize * num);
int ls = (num == 0 && StringUtils.isNotEmpty(lS)) ? Integer.parseInt(lS) : pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit:[pNo:{},pSize:{}]", pSize * num, pSize);
log.info("DM.execute.limit {},{} ==> begin", ls, pSize);
List<AdamAddresses> addressesList = new ArrayList<>();
List<AdamAddressesVo> vos = new ArrayList<>();
while (row.next()) {
String uid = row.getString(2);
......@@ -84,11 +83,17 @@ public class DMAddressesProcessor extends DataMigrationProcessorService {
AdamAddressesVo vo = AdamAddressesVo.getNew().copy(addresses);
addressesMapper.insert(addresses);
mongoTemplate.insert(vo, AdamAddressesVo.class.getSimpleName());
// addressesMapper.insert(addresses);
// mongoTemplate.insert(vo, AdamAddressesVo.class.getSimpleName());
addressesList.add(addresses);
vos.add(vo);
}
}
log.info("DM.execute.limit {},{} ==> handle.counts:{}", ls, pSize, addressesList.size());
if (!CollectionUtils.isEmpty(addressesList)) {
mongoTemplate.insert(vos, AdamAddressesVo.class.getSimpleName());
log.info("DM.execute.result:{}", adamAddressesService.saveBatch(addressesList));
}
num++;
ct -= pSize;
......
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMEntersProcessor.java
View file @ 5f5b421b
package com.liquidnet.service.adam.service.processor;
import com.liquidnet.common.cache.redis.util.RedisUtil;
import com.liquidnet.commons.lang.util.ServletUtils;
import com.liquidnet.service.adam.dto.vo.AdamAddressesVo;
import com.liquidnet.service.adam.dto.vo.AdamEntersVo;
import com.liquidnet.service.adam.entity.AdamEnters;
import com.liquidnet.service.adam.mapper.AdamEntersMapper;
import com.liquidnet.service.adam.service.DataMigrationProcessorService;
import com.liquidnet.service.adam.service.IAdamEntersService;
import com.mysql.cj.jdbc.result.ResultSetImpl;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import javax.servlet.http.HttpServletRequest;
import java.util.ArrayList;
import java.util.List;
@Slf4j
@Service
public class DMEntersProcessor extends DataMigrationProcessorService {
@Autowired
MongoTemplate mongoTemplate;
@Autowired
RedisUtil redisUtil;
@Autowired
AdamEntersMapper entersMapper;
IAdamEntersService adamEntersService;
@SneakyThrows
@Override
protected void dataProcessing() {
HttpServletRequest request = ServletUtils.getRequest();
String flg = StringUtils.defaultIfBlank(request.getParameter(DM_FLG), "");
String lS = request.getParameter(DM_LS), flg = StringUtils.defaultIfBlank(request.getParameter(DM_FLG), "");
if (StringUtils.isNotEmpty(request.getParameter(DM_FLUSH))) {
log.info("DM.flush.enters:{}", userMapper.executeForDM("adam_enters"));
log.info("DM.flush.AdamEntersVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").exists(true)), AdamEntersVo.class.getSimpleName()).getDeletedCount());
log.info("DM.flush.AdamEntersVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").ne(null)), AdamEntersVo.class.getSimpleName()).getDeletedCount());
}
String sqlCount = "select count(1) from enters where (type <> 1 or (type = 1 and is_certification = 'yes'))";
......@@ -56,15 +52,19 @@ public class DMEntersProcessor extends DataMigrationProcessorService {
row = (ResultSetImpl) statement.executeQuery();
row.first();
int ct = row.getInt(1), pSize = 1000, num = 0;
log.info("DM数据迁移总数:{}", ct);
log.info("DM.execute.counts:{}", ct);
while (ct > 0) {
statement = connection.prepareStatement(sql);
statement.setInt(1, pSize * num);
int ls = (num == 0 && StringUtils.isNotEmpty(lS)) ? Integer.parseInt(lS) : pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit:[pNo:{},pSize:{}]", pSize * num, pSize);
log.info("DM.execute.limit {},{} ==> begin", ls, pSize);
List<AdamEnters> entersList = new ArrayList<>();
List<AdamEntersVo> vos = new ArrayList<>();
while (row.next()) {
String uid = row.getString(2);
......@@ -82,11 +82,17 @@ public class DMEntersProcessor extends DataMigrationProcessorService {
AdamEntersVo vo = AdamEntersVo.getNew().copy(enters);
entersMapper.insert(enters);
mongoTemplate.insert(vo, AdamEntersVo.class.getSimpleName());
// entersMapper.insert(enters);
// mongoTemplate.insert(vo, AdamEntersVo.class.getSimpleName());
entersList.add(enters);
vos.add(vo);
}
}
log.info("DM.execute.limit {},{} ==> handle.counts:{}", ls, pSize, entersList.size());
if (!CollectionUtils.isEmpty(entersList)) {
mongoTemplate.insert(vos, AdamEntersVo.class.getSimpleName());
log.info("DM.execute.result:{}", adamEntersService.saveBatch(entersList));
}
num++;
ct -= pSize;
......
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMMemberOrderProcessor.java
View file @ 5f5b421b
package com.liquidnet.service.adam.service.processor;
import com.liquidnet.common.cache.redis.util.RedisUtil;
import com.liquidnet.commons.lang.util.ServletUtils;
import com.liquidnet.service.adam.dto.vo.AdamMemberOrderVo;
import com.liquidnet.service.adam.entity.AdamMemberOrder;
import com.liquidnet.service.adam.mapper.AdamMemberOrderMapper;
import com.liquidnet.service.adam.service.DataMigrationProcessorService;
import com.liquidnet.service.adam.service.IAdamMemberOrderService;
import com.mysql.cj.jdbc.result.ResultSetImpl;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import javax.servlet.http.HttpServletRequest;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Slf4j
@Service
public class DMMemberOrderProcessor extends DataMigrationProcessorService {
@Autowired
MongoTemplate mongoTemplate;
@Autowired
RedisUtil redisUtil;
@Autowired
AdamMemberOrderMapper memberOrderMapper;
IAdamMemberOrderService adamMemberOrderService;
public static final String MEMBER_NAME = "摩登天空会员";
public static final String MEMBER_ID = "1";
......@@ -58,11 +55,11 @@ public class DMMemberOrderProcessor extends DataMigrationProcessorService {
@Override
protected void dataProcessing() {
HttpServletRequest request = ServletUtils.getRequest();
String flg = StringUtils.defaultIfBlank(request.getParameter(DM_FLG), "");
String lS = request.getParameter(DM_LS), flg = StringUtils.defaultIfBlank(request.getParameter(DM_FLG), "");
if (StringUtils.isNotEmpty(request.getParameter(DM_FLUSH))) {
log.info("DM.flush.user_member:{}", userMapper.executeForDM("adam_member_order"));
log.info("DM.flush.AdamUserMemberVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").exists(true)), AdamMemberOrder.class.getSimpleName()).getDeletedCount());
log.info("DM.flush.AdamUserMemberVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").ne(null)), AdamMemberOrder.class.getSimpleName()).getDeletedCount());
}
String sqlCount = "select count(1) from user_vip_card_orders where status in (1,5)";
......@@ -82,15 +79,19 @@ public class DMMemberOrderProcessor extends DataMigrationProcessorService {
row = (ResultSetImpl) statement.executeQuery();
row.first();
int ct = row.getInt(1), pSize = 1000, num = 0;
log.info("DM数据迁移总数:{}", ct);
log.info("DM.execute.counts:{}", ct);
while (ct > 0) {
statement = connection.prepareStatement(sql);
statement.setInt(1, pSize * num);
int ls = (num == 0 && StringUtils.isNotEmpty(lS)) ? Integer.parseInt(lS) : pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit:[pNo:{},pSize:{}]", pSize * num, pSize);
log.info("DM.execute.limit {},{} ==> begin", ls, pSize);
List<AdamMemberOrder> memberOrderList = new ArrayList<>();
List<AdamMemberOrderVo> vos = new ArrayList<>();
while (row.next()) {
String uid = row.getString(1);
......@@ -121,12 +122,17 @@ public class DMMemberOrderProcessor extends DataMigrationProcessorService {
AdamMemberOrderVo vo = AdamMemberOrderVo.getNew().copy(memberOrder);
memberOrderMapper.insert(memberOrder);
mongoTemplate.insert(vo, AdamMemberOrderVo.class.getSimpleName());
// memberOrderMapper.insert(memberOrder);
// mongoTemplate.insert(vo, AdamMemberOrderVo.class.getSimpleName());
memberOrderList.add(memberOrder);
vos.add(vo);
}
}
log.info("DM.execute.limit {},{} ==> handle.counts:{}", ls, pSize, memberOrderList.size());
if (!CollectionUtils.isEmpty(memberOrderList)) {
mongoTemplate.insert(vos, AdamMemberOrderVo.class.getSimpleName());
log.info("DM.execute.result:{}", adamMemberOrderService.saveBatch(memberOrderList));
}
num++;
ct -= pSize;
......
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMRealNameProcessor.java
View file @ 5f5b421b
package com.liquidnet.service.adam.service.processor;
import com.liquidnet.common.cache.redis.util.RedisUtil;
import com.liquidnet.commons.lang.util.ServletUtils;
import com.liquidnet.service.adam.dto.vo.AdamRealInfoVo;
import com.liquidnet.service.adam.entity.AdamRealName;
import com.liquidnet.service.adam.mapper.AdamRealNameMapper;
import com.liquidnet.service.adam.service.DataMigrationProcessorService;
import com.liquidnet.service.adam.service.IAdamRealNameService;
import com.mysql.cj.jdbc.result.ResultSetImpl;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import javax.servlet.http.HttpServletRequest;
import java.util.ArrayList;
import java.util.List;
@Slf4j
@Service
public class DMRealNameProcessor extends DataMigrationProcessorService {
@Autowired
MongoTemplate mongoTemplate;
@Autowired
RedisUtil redisUtil;
@Autowired
AdamRealNameMapper realNameMapper;
IAdamRealNameService adamRealNameService;
@SneakyThrows
@Override
protected void dataProcessing() {
HttpServletRequest request = ServletUtils.getRequest();
String flg = StringUtils.defaultIfBlank(request.getParameter(DM_FLG), "");
String lS = request.getParameter(DM_LS), flg = StringUtils.defaultIfBlank(request.getParameter(DM_FLG), "");
if (StringUtils.isNotEmpty(request.getParameter(DM_FLUSH))) {
log.info("DM.flush.real_name:{}", userMapper.executeForDM("adam_real_name"));
log.info("DM.flush.AdamRealInfoVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").exists(true)), AdamRealInfoVo.class.getSimpleName()).getDeletedCount());
log.info("DM.flush.AdamRealInfoVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").ne(null)), AdamRealInfoVo.class.getSimpleName()).getDeletedCount());
}
String sqlCount = "select count(1) from user_real_name where 1=1";
......@@ -48,22 +45,24 @@ public class DMRealNameProcessor extends DataMigrationProcessorService {
sqlCount += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
sql += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
}
sql += " limit ?,?";
statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery();
row.first();
int ct = row.getInt(1), pSize = 1000, num = 0;
log.info("DM数据迁移总数:{}", ct);
log.info("DM.execute.counts:{}", ct);
while (ct > 0) {
statement = connection.prepareStatement(sql);
statement.setInt(1, pSize * num);
int ls = (num == 0 && StringUtils.isNotEmpty(lS)) ? Integer.parseInt(lS) : pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit:[pNo:{},pSize:{}]", pSize * num, pSize);
log.info("DM.execute.limit {},{} ==> begin", ls, pSize);
List<AdamRealName> realNameList = new ArrayList<>();
List<AdamRealInfoVo> vos = new ArrayList<>();
while (row.next()) {
String uid = row.getString(2);
......@@ -85,11 +84,17 @@ public class DMRealNameProcessor extends DataMigrationProcessorService {
vo.setIdCard(realName.getIdCard());
vo.setState(1);
realNameMapper.insert(realName);
mongoTemplate.insert(vo, AdamRealInfoVo.class.getSimpleName());
// realNameMapper.insert(realName);
// mongoTemplate.insert(vo, AdamRealInfoVo.class.getSimpleName());
realNameList.add(realName);
vos.add(vo);
}
}
log.info("DM.execute.limit {},{} ==> handle.counts:{}", ls, pSize, realNameList.size());
if (!CollectionUtils.isEmpty(realNameList)) {
mongoTemplate.insert(vos, AdamRealInfoVo.class.getSimpleName());
log.info("DM.execute.result:{}", adamRealNameService.saveBatch(realNameList));
}
num++;
ct -= pSize;
......
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMThirdPartsProcessor.java
View file @ 5f5b421b
package com.liquidnet.service.adam.service.processor;
import com.liquidnet.common.cache.redis.util.RedisUtil;
import com.liquidnet.commons.lang.util.ServletUtils;
import com.liquidnet.service.adam.dto.vo.AdamThirdPartInfoVo;
import com.liquidnet.service.adam.entity.AdamThirdParty;
import com.liquidnet.service.adam.mapper.AdamThirdPartyMapper;
import com.liquidnet.service.adam.service.DataMigrationProcessorService;
import com.liquidnet.service.adam.service.IAdamThirdPartyService;
import com.mysql.cj.jdbc.result.ResultSetImpl;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import javax.servlet.http.HttpServletRequest;
import java.util.ArrayList;
import java.util.List;
@Slf4j
@Service
public class DMThirdPartsProcessor extends DataMigrationProcessorService {
@Autowired
MongoTemplate mongoTemplate;
@Autowired
RedisUtil redisUtil;
@Autowired
AdamThirdPartyMapper thirdPartyMapper;
IAdamThirdPartyService adamThirdPartyService;
@SneakyThrows
@Override
protected void dataProcessing() {
HttpServletRequest request = ServletUtils.getRequest();
String flg = StringUtils.defaultIfBlank(request.getParameter(DM_FLG), "");
String lS = request.getParameter(DM_LS), flg = StringUtils.defaultIfBlank(request.getParameter(DM_FLG), "");
if (StringUtils.isNotEmpty(request.getParameter(DM_FLUSH))) {
log.info("DM.flush.third_party:{}", userMapper.executeForDM("adam_third_party"));
log.info("DM.flush.AdamThirdPartInfoVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").exists(true)), AdamThirdPartInfoVo.class.getSimpleName()).getDeletedCount());
log.info("DM.flush.AdamThirdPartInfoVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").ne(null)), AdamThirdPartInfoVo.class.getSimpleName()).getDeletedCount());
}
String sqlCount = "select count(1) from user_third_parts where `status`=1 and delete_tag=0";
......@@ -55,16 +52,19 @@ public class DMThirdPartsProcessor extends DataMigrationProcessorService {
row = (ResultSetImpl) statement.executeQuery();
row.first();
int ct = row.getInt(1), pSize = 1000, num = 0;
log.info("DM数据迁移总数:{}", ct);
log.info("DM.execute.counts:{}", ct);
while (ct > 0) {
statement = connection.prepareStatement(sql);
statement.setInt(1, pSize * num);
int ls = (num == 0 && StringUtils.isNotEmpty(lS)) ? Integer.parseInt(lS) : pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit:[pNo:{},pSize:{}]", pSize * num, pSize);
log.info("DM.execute.limit {},{} ==> begin", ls, pSize);
List<AdamThirdParty> thirdPartyList = new ArrayList<>();
List<AdamThirdPartInfoVo> vos = new ArrayList<>();
while (row.next()) {
String uid = row.getString(1);
......@@ -89,11 +89,17 @@ public class DMThirdPartsProcessor extends DataMigrationProcessorService {
vo.setCreatedAt(thirdParty.getCreatedAt());
vo.setUpdatedAt(thirdParty.getUpdatedAt());
thirdPartyMapper.insert(thirdParty);
mongoTemplate.insert(vo, AdamThirdPartInfoVo.class.getSimpleName());
// thirdPartyMapper.insert(thirdParty);
// mongoTemplate.insert(vo, AdamThirdPartInfoVo.class.getSimpleName());
thirdPartyList.add(thirdParty);
vos.add(vo);
}
}
log.info("DM.execute.limit {},{} ==> handle.counts:{}", ls, pSize, thirdPartyList.size());
if (!CollectionUtils.isEmpty(thirdPartyList)) {
mongoTemplate.insert(vos, AdamThirdPartInfoVo.class.getSimpleName());
log.info("DM.execute.result:{}", adamThirdPartyService.saveBatch(thirdPartyList));
}
num++;
ct -= pSize;
......
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMUserInformationProcessor.java
View file @ 5f5b421b
package com.liquidnet.service.adam.service.processor;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.liquidnet.common.cache.redis.util.RedisUtil;
import com.liquidnet.commons.lang.util.DateUtil;
import com.liquidnet.commons.lang.util.JsonUtils;
import com.liquidnet.commons.lang.util.ServletUtils;
......@@ -9,32 +8,34 @@ import com.liquidnet.service.adam.dto.vo.AdamTagVo;
import com.liquidnet.service.adam.dto.vo.AdamUserInfoVo;
import com.liquidnet.service.adam.entity.AdamUser;
import com.liquidnet.service.adam.entity.AdamUserInfo;
import com.liquidnet.service.adam.mapper.AdamUserInfoMapper;
import com.liquidnet.service.adam.service.DataMigrationProcessorService;
import com.liquidnet.service.adam.service.IAdamUserInfoService;
import com.liquidnet.service.adam.service.IAdamUserService;
import com.mysql.cj.jdbc.result.ResultSetImpl;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import javax.servlet.http.HttpServletRequest;
import java.sql.SQLException;
import java.util.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Slf4j
@Service
public class DMUserInformationProcessor extends DataMigrationProcessorService {
@Autowired
MongoTemplate mongoTemplate;
IAdamUserService adamUserService;
@Autowired
RedisUtil redisUtil;
@Autowired
AdamUserInfoMapper userInfoMapper;
IAdamUserInfoService adamUserInfoService;
public static final String DM_FAILED_USER = "dm:failed:uid";
public static final String DM_EXISTS_USER = "dm:exists:uid";
......@@ -49,39 +50,41 @@ public class DMUserInformationProcessor extends DataMigrationProcessorService {
@Override
protected void dataProcessing() {
HttpServletRequest request = ServletUtils.getRequest();
String flg = StringUtils.defaultIfBlank(request.getParameter(DM_FLG), "");
String lS = request.getParameter(DM_LS), flg = StringUtils.defaultIfBlank(request.getParameter(DM_FLG), "");
if (StringUtils.isNotEmpty(request.getParameter(DM_FLUSH))) {
log.info("DM.flush.user:{}", userMapper.executeForDM("adam_user"));
log.info("DM.flush.user_info:{}", userMapper.executeForDM("adam_user_info"));
log.info("DM.flush.AdamUserInfoVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").exists(true)), AdamUserInfoVo.class.getSimpleName()).getDeletedCount());
log.info("DM.flush.AdamUserInfoVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").ne(null)), AdamUserInfoVo.class.getSimpleName()).getDeletedCount());
}
String sqlCount = "select count(1) from users where delete_tag in (0,1) and length(mobile)=11 ";
String field = "uid,mobile,username,sex,birthday,area,sign,face_url,back_url,created_at,updated_at,is_complete,rong_cloud_token,rong_cloud_tag";
// 1-uid,2-mobile,3-username,4-sex,5-birthday,6-area,7-sign,8-face_url,9-back_url,10-created_at,11-updated_at,12-is_complete,13-rong_cloud_token,14-rong_cloud_tag
String sql = sqlCount.replace("count(1)", field);
if ("1".equals(flg)) {
sqlCount += "and mobile in ('13753596360','17701223310','18518013286','18548596019','15901093014','17382319723','13631444097','13671737873')";
sql += " and mobile in ('13753596360','17701223310','18518013286','18548596019','15901093014','17382319723','13631444097','13671737873')";
}
sql += " limit ?,?";
statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery();
row.first();
int ct = row.getInt(1), pSize = 1000, num = 0;
log.info("DM数据迁移总数:{}", ct);
log.info("DM.execute.counts:{}", ct);
while (ct > 0) {
statement = connection.prepareStatement(sql);
statement.setInt(1, pSize * num);
int ls = (num == 0 && StringUtils.isNotEmpty(lS)) ? Integer.parseInt(lS) : pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit:[pNo:{},pSize:{}]", pSize * num, pSize);
log.info("DM.execute.limit {},{} ==> begin", ls, pSize);
List<AdamUser> userList = new ArrayList<>();
List<AdamUserInfo> userInfoList = new ArrayList<>();
List<AdamUserInfoVo> vos = new ArrayList<>();
while (row.next()) {
try {
AdamTagVo sex = sexMap.get(row.getString(4));
......@@ -129,9 +132,12 @@ public class DMUserInformationProcessor extends DataMigrationProcessorService {
vo.setIsComplete(row.getInt(12));
vo.setQrCode(userInfo.getQrCode());
userMapper.insert(user);
userInfoMapper.insert(userInfo);
mongoTemplate.insert(vo, AdamUserInfoVo.class.getSimpleName());
// userMapper.insert(user);
// userInfoMapper.insert(userInfo);
// mongoTemplate.insert(vo, AdamUserInfoVo.class.getSimpleName());
userList.add(user);
userInfoList.add(userInfo);
vos.add(vo);
}
}
} catch (SQLException throwables) {
......@@ -140,7 +146,12 @@ public class DMUserInformationProcessor extends DataMigrationProcessorService {
throwables.getErrorCode(), throwables.getMessage());
}
}
log.info("DM.execute.limit {},{} ==> handle.counts:{}", ls, pSize, userList.size());
if (!CollectionUtils.isEmpty(userList)) {
mongoTemplate.insert(vos, AdamUserInfoVo.class.getSimpleName());
log.info("DM.execute.result:{}", adamUserService.saveBatch(userList));
log.info("DM.execute.result:{}", adamUserInfoService.saveBatch(userInfoList));
}
num++;
ct -= pSize;
......
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMUserMemberProcessor.java
View file @ 5f5b421b
package com.liquidnet.service.adam.service.processor;
import com.liquidnet.common.cache.redis.util.RedisUtil;
import com.liquidnet.commons.lang.util.ServletUtils;
import com.liquidnet.service.adam.dto.vo.AdamUserInfoVo;
import com.liquidnet.service.adam.dto.vo.AdamUserMemberVo;
import com.liquidnet.service.adam.entity.AdamUserMember;
import com.liquidnet.service.adam.mapper.AdamUserMemberMapper;
import com.liquidnet.service.adam.service.DataMigrationProcessorService;
import com.liquidnet.service.adam.service.IAdamUserMemberService;
import com.mysql.cj.jdbc.result.ResultSetImpl;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import javax.servlet.http.HttpServletRequest;
import java.util.ArrayList;
import java.util.List;
@Slf4j
@Service
public class DMUserMemberProcessor extends DataMigrationProcessorService {
@Autowired
MongoTemplate mongoTemplate;
@Autowired
RedisUtil redisUtil;
@Autowired
AdamUserMemberMapper userMemberMapper;
IAdamUserMemberService adamUserMemberService;
@SneakyThrows
@Override
protected void dataProcessing() {
HttpServletRequest request = ServletUtils.getRequest();
String flg = StringUtils.defaultIfBlank(request.getParameter(DM_FLG), "");
String lS = request.getParameter(DM_LS), flg = StringUtils.defaultIfBlank(request.getParameter(DM_FLG), "");
if (StringUtils.isNotEmpty(request.getParameter(DM_FLUSH))) {
log.info("DM.flush.user_member:{}", userMapper.executeForDM("adam_user_member"));
log.info("DM.flush.AdamUserMemberVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").exists(true)), AdamUserMemberVo.class.getSimpleName()).getDeletedCount());
log.info("DM.flush.AdamUserMemberVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").ne(null)), AdamUserMemberVo.class.getSimpleName()).getDeletedCount());
}
String sqlCount = "select count(1) from user_cards where 1=1";
......@@ -56,15 +52,18 @@ public class DMUserMemberProcessor extends DataMigrationProcessorService {
row = (ResultSetImpl) statement.executeQuery();
row.first();
int ct = row.getInt(1), pSize = 1000, num = 0;
log.info("DM数据迁移总数:{}", ct);
log.info("DM.execute.counts:{}", ct);
while (ct > 0) {
statement = connection.prepareStatement(sql);
statement.setInt(1, pSize * num);
int ls = (num == 0 && StringUtils.isNotEmpty(lS)) ? Integer.parseInt(lS) : pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit:[pNo:{},pSize:{}]", pSize * num, pSize);
log.info("DM.execute.limit {},{} ==> begin", ls, pSize);
List<AdamUserMember> userMemberList = new ArrayList<>();
List<AdamUserMemberVo> vos = new ArrayList<>();
while (row.next()) {
String uid = row.getString(1);
......@@ -87,11 +86,17 @@ public class DMUserMemberProcessor extends DataMigrationProcessorService {
vo.setCreatedAt(userMember.getCreatedAt());
vo.setUpdatedAt(userMember.getUpdatedAt());
userMemberMapper.insert(userMember);
mongoTemplate.insert(vo, AdamUserMemberVo.class.getSimpleName());
// userMemberMapper.insert(userMember);
// mongoTemplate.insert(vo, AdamUserMemberVo.class.getSimpleName());
userMemberList.add(userMember);
vos.add(vo);
}
}
log.info("DM.execute.limit {},{} ==> handle.counts:{}", ls, pSize, userMemberList.size());
if (!CollectionUtils.isEmpty(userMemberList)) {
mongoTemplate.insert(vos, AdamUserMemberVo.class.getSimpleName());
log.info("DM.execute.result:{}", adamUserMemberService.saveBatch(userMemberList));
}
num++;
ct -= pSize;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment