记得上下班打卡 | git大法好,push需谨慎

Commit 0d4f0d68 authored by 张国柄's avatar 张国柄

DM;

parent 3a424d6b
...@@ -18,6 +18,7 @@ import java.util.Arrays; ...@@ -18,6 +18,7 @@ import java.util.Arrays;
import java.util.List; import java.util.List;
public abstract class DataMigrationProcessorService { public abstract class DataMigrationProcessorService {
public static final String DM_INCR_DT = "incrDt";
public static final String DM_LS = "lS"; public static final String DM_LS = "lS";
public static final String DM_FLG = "flg"; public static final String DM_FLG = "flg";
public static final String DM_FLUSH = "flu"; public static final String DM_FLUSH = "flu";
......
...@@ -29,13 +29,14 @@ public class DMAddressesProcessor extends DataMigrationProcessorService { ...@@ -29,13 +29,14 @@ public class DMAddressesProcessor extends DataMigrationProcessorService {
@Override @Override
protected void dataProcessing() { protected void dataProcessing() {
HttpServletRequest request = ServletUtils.getRequest(); HttpServletRequest request = ServletUtils.getRequest();
String lS = request.getParameter(DM_LS), flg = StringUtils.defaultIfBlank(request.getParameter(DM_FLG), ""); String lS = request.getParameter(DM_LS), incrDt = request.getParameter(DM_INCR_DT),
flg = StringUtils.defaultIfBlank(request.getParameter(DM_FLG), "");
if (StringUtils.isNotEmpty(request.getParameter(DM_FLUSH))) { if (StringUtils.isNotEmpty(request.getParameter(DM_FLUSH))) {
log.info("DM.flush.addresses:{}", userMapper.executeForDM("adam_addresses")); log.info("DM.flush.addresses:{}", userMapper.executeForDM("adam_addresses"));
log.info("DM.flush.AdamAddressesVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").exists(true)), AdamAddressesVo.class.getSimpleName()).getDeletedCount()); log.info("DM.flush.AdamAddressesVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").exists(true)), AdamAddressesVo.class.getSimpleName()).getDeletedCount());
} }
String sqlCount = "select count(1) from addresses where 1=1"; String sqlCount = "select count(1) from addresses where created_at" + (StringUtils.isBlank(incrDt) ? "<" : ">=") + "curdate()";
String field = "id,user_id,user_name,user_phone,province,city,county,user_address,defaultDelivery,created_at"; String field = "id,user_id,user_name,user_phone,province,city,county,user_address,defaultDelivery,created_at";
// 1-id,2-user_id,3-user_name,4-user_phone,5-province,6-city,7-county,8-user_address,9-defaultDelivery,10-created_at // 1-id,2-user_id,3-user_name,4-user_phone,5-province,6-city,7-county,8-user_address,9-defaultDelivery,10-created_at
String sql = sqlCount.replace("count(1)", field); String sql = sqlCount.replace("count(1)", field);
...@@ -43,13 +44,13 @@ public class DMAddressesProcessor extends DataMigrationProcessorService { ...@@ -43,13 +44,13 @@ public class DMAddressesProcessor extends DataMigrationProcessorService {
sqlCount += " and user_id in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')"; sqlCount += " and user_id in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
sql += " and user_id in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')"; sql += " and user_id in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
} }
sql += " limit ?,?"; sql += " order by id limit ?,?";
statement = connection.prepareStatement(sqlCount); statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
row.first(); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct; int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start", pl, num, pSize, tl); log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
while (ct > 0) { while (ct > 0) {
statement = connection.prepareStatement(sql); statement = connection.prepareStatement(sql);
......
...@@ -29,14 +29,15 @@ public class DMCollectionProcessor extends DataMigrationProcessorService { ...@@ -29,14 +29,15 @@ public class DMCollectionProcessor extends DataMigrationProcessorService {
@Override @Override
protected void dataProcessing() { protected void dataProcessing() {
HttpServletRequest request = ServletUtils.getRequest(); HttpServletRequest request = ServletUtils.getRequest();
String lS = request.getParameter(DM_LS), flg = StringUtils.defaultIfBlank(request.getParameter(DM_FLG), ""); String lS = request.getParameter(DM_LS), incrDt = request.getParameter(DM_INCR_DT),
flg = StringUtils.defaultIfBlank(request.getParameter(DM_FLG), "");
if (StringUtils.isNotEmpty(request.getParameter(DM_FLUSH))) { if (StringUtils.isNotEmpty(request.getParameter(DM_FLUSH))) {
log.info("DM.flush.collection:{}", userMapper.executeForDM("adam_collection")); log.info("DM.flush.collection:{}", userMapper.executeForDM("adam_collection"));
log.info("DM.flush.AdamCollectVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").exists(true)), AdamCollectVo.class.getSimpleName()).getDeletedCount()); log.info("DM.flush.AdamCollectVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").exists(true)), AdamCollectVo.class.getSimpleName()).getDeletedCount());
} }
String sqlCount = "select count(1) from user_collections where type='TICKET' and status=1 and content_id>=5722"; String sqlCount = "select count(1) from user_collections where type='TICKET' and status=1 and content_id>=5722 and created_at"+(StringUtils.isBlank(incrDt) ? "<" : ">=")+"curdate()";
String field = "uid,content_id,`type`,`status`,created_at,updated_at"; String field = "uid,content_id,`type`,`status`,created_at,updated_at";
// 1-uid,2-content_id,3-`type`,4-`status`,5-created_at,6-updated_at // 1-uid,2-content_id,3-`type`,4-`status`,5-created_at,6-updated_at
String sql = sqlCount.replace("count(1)", field); String sql = sqlCount.replace("count(1)", field);
...@@ -44,13 +45,13 @@ public class DMCollectionProcessor extends DataMigrationProcessorService { ...@@ -44,13 +45,13 @@ public class DMCollectionProcessor extends DataMigrationProcessorService {
sqlCount += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')"; sqlCount += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
sql += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')"; sql += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
} }
sql += " limit ?,?"; sql += " order by id limit ?,?";
statement = connection.prepareStatement(sqlCount); statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
row.first(); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct; int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start", pl, num, pSize, tl); log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
while (ct > 0) { while (ct > 0) {
statement = connection.prepareStatement(sql); statement = connection.prepareStatement(sql);
......
...@@ -29,14 +29,15 @@ public class DMEntersProcessor extends DataMigrationProcessorService { ...@@ -29,14 +29,15 @@ public class DMEntersProcessor extends DataMigrationProcessorService {
@Override @Override
protected void dataProcessing() { protected void dataProcessing() {
HttpServletRequest request = ServletUtils.getRequest(); HttpServletRequest request = ServletUtils.getRequest();
String lS = request.getParameter(DM_LS), flg = StringUtils.defaultIfBlank(request.getParameter(DM_FLG), ""); String lS = request.getParameter(DM_LS), incrDt = request.getParameter(DM_INCR_DT),
flg = StringUtils.defaultIfBlank(request.getParameter(DM_FLG), "");
if (StringUtils.isNotEmpty(request.getParameter(DM_FLUSH))) { if (StringUtils.isNotEmpty(request.getParameter(DM_FLUSH))) {
log.info("DM.flush.enters:{}", userMapper.executeForDM("adam_enters")); log.info("DM.flush.enters:{}", userMapper.executeForDM("adam_enters"));
log.info("DM.flush.AdamEntersVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").exists(true)), AdamEntersVo.class.getSimpleName()).getDeletedCount()); log.info("DM.flush.AdamEntersVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").exists(true)), AdamEntersVo.class.getSimpleName()).getDeletedCount());
} }
String sqlCount = "select count(1) from enters where (type <> 1 or (type = 1 and is_certification = 'yes'))"; String sqlCount = "select count(1) from enters where (type <> 1 or (type = 1 and is_certification = 'yes')) and created_at" + (StringUtils.isBlank(incrDt) ? "<" : ">=") + "curdate()";
String field = "id,user_id,type,`name`,mobile,idcode,if(is_default='no', 0, 1),created_at"; String field = "id,user_id,type,`name`,mobile,idcode,if(is_default='no', 0, 1),created_at";
// 1-id,2-user_id,3-type,4-`name`,5-mobile,6-idcode,7-if(is_default='no', 0, 1),8-created_at // 1-id,2-user_id,3-type,4-`name`,5-mobile,6-idcode,7-if(is_default='no', 0, 1),8-created_at
String sql = sqlCount.replace("count(1)", field); String sql = sqlCount.replace("count(1)", field);
...@@ -44,13 +45,13 @@ public class DMEntersProcessor extends DataMigrationProcessorService { ...@@ -44,13 +45,13 @@ public class DMEntersProcessor extends DataMigrationProcessorService {
sqlCount += " and user_id in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')"; sqlCount += " and user_id in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
sql += " and user_id in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')"; sql += " and user_id in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
} }
sql += " limit ?,?"; sql += " order by id limit ?,?";
statement = connection.prepareStatement(sqlCount); statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
row.first(); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct; int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start", pl, num, pSize, tl); log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
while (ct > 0) { while (ct > 0) {
statement = connection.prepareStatement(sql); statement = connection.prepareStatement(sql);
......
...@@ -182,7 +182,8 @@ public class DMMemberCodeProcessor extends DataMigrationProcessorService { ...@@ -182,7 +182,8 @@ public class DMMemberCodeProcessor extends DataMigrationProcessorService {
@Override @Override
protected void dataProcessing() { protected void dataProcessing() {
HttpServletRequest request = ServletUtils.getRequest(); HttpServletRequest request = ServletUtils.getRequest();
String lS = request.getParameter(DM_LS), flg = StringUtils.defaultIfBlank(request.getParameter(DM_FLG), ""); String lS = request.getParameter(DM_LS), incrDt = request.getParameter(DM_INCR_DT),
flg = StringUtils.defaultIfBlank(request.getParameter(DM_FLG), "");
if (StringUtils.isNotEmpty(request.getParameter(DM_FLUSH))) { if (StringUtils.isNotEmpty(request.getParameter(DM_FLUSH))) {
log.info("DM.flush.member_code:{}", userMapper.executeForDM("adam_member_code")); log.info("DM.flush.member_code:{}", userMapper.executeForDM("adam_member_code"));
...@@ -194,13 +195,13 @@ public class DMMemberCodeProcessor extends DataMigrationProcessorService { ...@@ -194,13 +195,13 @@ public class DMMemberCodeProcessor extends DataMigrationProcessorService {
// 1-vip_card_code,2-`type`+1,3-card_number,4-`status`,5-created_at,6-updated_at, // 1-vip_card_code,2-`type`+1,3-card_number,4-`status`,5-created_at,6-updated_at,
// 7-buy_order_id,8-buy_uid,9-buy_time,10-use_order_id,11-use_uid,12-use_time // 7-buy_order_id,8-buy_uid,9-buy_time,10-use_order_id,11-use_uid,12-use_time
String sql = sqlCount.replace("count(1)", field); String sql = sqlCount.replace("count(1)", field);
sql += " limit ?,?"; sql += " order by id limit ?,?";
statement = connection.prepareStatement(sqlCount); statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
row.first(); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct; int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start", pl, num, pSize, tl); log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
while (ct > 0) { while (ct > 0) {
statement = connection.prepareStatement(sql); statement = connection.prepareStatement(sql);
......
...@@ -30,6 +30,7 @@ public class DMMemberOrderProcessor extends DataMigrationProcessorService { ...@@ -30,6 +30,7 @@ public class DMMemberOrderProcessor extends DataMigrationProcessorService {
// APPWEPAY|APPALIPAY|WAPWEPAY|WAPALIPAY|JSWEPAY|GIFT_CODE|VIP_CODE|APPLETWEPAY // APPWEPAY|APPALIPAY|WAPWEPAY|WAPALIPAY|JSWEPAY|GIFT_CODE|VIP_CODE|APPLETWEPAY
public static final Map<String, String> dfMap = new HashMap<>(); public static final Map<String, String> dfMap = new HashMap<>();
public static final Map<String, String> ptMap = new HashMap<>(); public static final Map<String, String> ptMap = new HashMap<>();
static { static {
dfMap.put("APPWEPAY", "app"); dfMap.put("APPWEPAY", "app");
dfMap.put("APPALIPAY", "app"); dfMap.put("APPALIPAY", "app");
...@@ -52,14 +53,15 @@ public class DMMemberOrderProcessor extends DataMigrationProcessorService { ...@@ -52,14 +53,15 @@ public class DMMemberOrderProcessor extends DataMigrationProcessorService {
@Override @Override
protected void dataProcessing() { protected void dataProcessing() {
HttpServletRequest request = ServletUtils.getRequest(); HttpServletRequest request = ServletUtils.getRequest();
String lS = request.getParameter(DM_LS), flg = StringUtils.defaultIfBlank(request.getParameter(DM_FLG), ""); String lS = request.getParameter(DM_LS), incrDt = request.getParameter(DM_INCR_DT),
flg = StringUtils.defaultIfBlank(request.getParameter(DM_FLG), "");
if (StringUtils.isNotEmpty(request.getParameter(DM_FLUSH))) { if (StringUtils.isNotEmpty(request.getParameter(DM_FLUSH))) {
log.info("DM.flush.member_order:{}", userMapper.executeForDM("adam_member_order")); log.info("DM.flush.member_order:{}", userMapper.executeForDM("adam_member_order"));
log.info("DM.flush.AdamMemberOrderVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").exists(true)), AdamMemberOrderVo.class.getSimpleName()).getDeletedCount()); log.info("DM.flush.AdamMemberOrderVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").exists(true)), AdamMemberOrderVo.class.getSimpleName()).getDeletedCount());
} }
String sqlCount = "select count(1) from user_vip_card_orders where status in (1,5)"; String sqlCount = "select count(1) from user_vip_card_orders where status in (1,5) and created_at" + (StringUtils.isBlank(incrDt) ? "<" : ">=") + "curdate()";
String field = "order_code,uid,order_sub_type,vip_card_price,price,status,card_number,birthday,pay_type,pay_code,pay_at,created_at,updated_at,client_ip,source,version"; String field = "order_code,uid,order_sub_type,vip_card_price,price,status,card_number,birthday,pay_type,pay_code,pay_at,created_at,updated_at,client_ip,source,version";
// 1-order_code,2-uid,3-order_sub_type,4-vip_card_price,5-price,6-status,7-card_number,8-birthday, // 1-order_code,2-uid,3-order_sub_type,4-vip_card_price,5-price,6-status,7-card_number,8-birthday,
// 9-pay_type,10-pay_code,11-pay_at,12-created_at,13-updated_at,14-client_ip,15-source,16-version // 9-pay_type,10-pay_code,11-pay_at,12-created_at,13-updated_at,14-client_ip,15-source,16-version
...@@ -68,13 +70,13 @@ public class DMMemberOrderProcessor extends DataMigrationProcessorService { ...@@ -68,13 +70,13 @@ public class DMMemberOrderProcessor extends DataMigrationProcessorService {
sqlCount += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')"; sqlCount += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
sql += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')"; sql += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
} }
sql += " limit ?,?"; sql += " order by id limit ?,?";
statement = connection.prepareStatement(sqlCount); statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
row.first(); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct; int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start", pl, num, pSize, tl); log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
while (ct > 0) { while (ct > 0) {
statement = connection.prepareStatement(sql); statement = connection.prepareStatement(sql);
......
...@@ -29,14 +29,15 @@ public class DMRealNameProcessor extends DataMigrationProcessorService { ...@@ -29,14 +29,15 @@ public class DMRealNameProcessor extends DataMigrationProcessorService {
@Override @Override
protected void dataProcessing() { protected void dataProcessing() {
HttpServletRequest request = ServletUtils.getRequest(); HttpServletRequest request = ServletUtils.getRequest();
String lS = request.getParameter(DM_LS), flg = StringUtils.defaultIfBlank(request.getParameter(DM_FLG), ""); String lS = request.getParameter(DM_LS), incrDt = request.getParameter(DM_INCR_DT),
flg = StringUtils.defaultIfBlank(request.getParameter(DM_FLG), "");
if (StringUtils.isNotEmpty(request.getParameter(DM_FLUSH))) { if (StringUtils.isNotEmpty(request.getParameter(DM_FLUSH))) {
log.info("DM.flush.real_name:{}", userMapper.executeForDM("adam_real_name")); log.info("DM.flush.real_name:{}", userMapper.executeForDM("adam_real_name"));
log.info("DM.flush.AdamRealInfoVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").exists(true)), AdamRealInfoVo.class.getSimpleName()).getDeletedCount()); log.info("DM.flush.AdamRealInfoVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").exists(true)), AdamRealInfoVo.class.getSimpleName()).getDeletedCount());
} }
String sqlCount = "select count(1) from user_real_name where 1=1"; String sqlCount = "select count(1) from user_real_name where id not in (9145,7456,7448,7197,7194,4137,4138,4139,269,261,262,263,264,265,266,267,251) and created_at" + (StringUtils.isBlank(incrDt) ? "<" : ">=") + "curdate()";
String field = "id,uid,real_name,id_card,ifnull(created_at, '1970-01-01 00:00:00') created_at,updated_at"; String field = "id,uid,real_name,id_card,ifnull(created_at, '1970-01-01 00:00:00') created_at,updated_at";
// 1-id,2-uid,3-real_name,4-id_card,5-created_at,6-updated_at // 1-id,2-uid,3-real_name,4-id_card,5-created_at,6-updated_at
String sql = sqlCount.replace("count(1)", field); String sql = sqlCount.replace("count(1)", field);
...@@ -44,13 +45,13 @@ public class DMRealNameProcessor extends DataMigrationProcessorService { ...@@ -44,13 +45,13 @@ public class DMRealNameProcessor extends DataMigrationProcessorService {
sqlCount += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')"; sqlCount += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
sql += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')"; sql += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
} }
sql += " limit ?,?"; sql += " order by id limit ?,?";
statement = connection.prepareStatement(sqlCount); statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
row.first(); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct; int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start", pl, num, pSize, tl); log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
while (ct > 0) { while (ct > 0) {
statement = connection.prepareStatement(sql); statement = connection.prepareStatement(sql);
......
...@@ -29,7 +29,8 @@ public class DMThirdPartsProcessor extends DataMigrationProcessorService { ...@@ -29,7 +29,8 @@ public class DMThirdPartsProcessor extends DataMigrationProcessorService {
@Override @Override
protected void dataProcessing() { protected void dataProcessing() {
HttpServletRequest request = ServletUtils.getRequest(); HttpServletRequest request = ServletUtils.getRequest();
String lS = request.getParameter(DM_LS), flg = StringUtils.defaultIfBlank(request.getParameter(DM_FLG), ""); String lS = request.getParameter(DM_LS), incrDt = request.getParameter(DM_INCR_DT),
flg = StringUtils.defaultIfBlank(request.getParameter(DM_FLG), "");
if (StringUtils.isNotEmpty(request.getParameter(DM_FLUSH))) { if (StringUtils.isNotEmpty(request.getParameter(DM_FLUSH))) {
log.info("DM.flush.third_party:{}", userMapper.executeForDM("adam_third_party")); log.info("DM.flush.third_party:{}", userMapper.executeForDM("adam_third_party"));
...@@ -37,8 +38,8 @@ public class DMThirdPartsProcessor extends DataMigrationProcessorService { ...@@ -37,8 +38,8 @@ public class DMThirdPartsProcessor extends DataMigrationProcessorService {
} }
String sqlCount = "select count(1) from (\n" + String sqlCount = "select count(1) from (\n" +
" select row_number() over (partition by uid,type order by created_at desc) rn,uid,open_id,avatar,nickname,type,created_at,updated_at from user_third_parts where `status`=1 and delete_tag in (0,1) and open_id<>'' and uid>0\n" + " select row_number() over (partition by uid,type order by created_at desc) rn,id,uid,open_id,avatar,nickname,type,created_at,updated_at from user_third_parts where `status`=1 and delete_tag in (0,1) and open_id<>'' and uid>0 and created_at"+(StringUtils.isBlank(incrDt) ? "<" : ">=")+"curdate()\n" +
" ) t where t.rn = 1"; " ) t where t.rn=1";
String field = "uid,open_id,avatar,nickname,type,created_at,updated_at"; String field = "uid,open_id,avatar,nickname,type,created_at,updated_at";
// 1-uid,2-open_id,3-avatar,4-nickname,5-type,6-created_at,7-updated_at // 1-uid,2-open_id,3-avatar,4-nickname,5-type,6-created_at,7-updated_at
String sql = sqlCount.replace("count(1)", field); String sql = sqlCount.replace("count(1)", field);
...@@ -46,13 +47,13 @@ public class DMThirdPartsProcessor extends DataMigrationProcessorService { ...@@ -46,13 +47,13 @@ public class DMThirdPartsProcessor extends DataMigrationProcessorService {
sqlCount += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')"; sqlCount += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
sql += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')"; sql += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
} }
sql += " limit ?,?"; sql += " order by id limit ?,?";
statement = connection.prepareStatement(sqlCount); statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
row.first(); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct; int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start", pl, num, pSize, tl); log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
while (ct > 0) { while (ct > 0) {
statement = connection.prepareStatement(sql); statement = connection.prepareStatement(sql);
......
...@@ -40,6 +40,7 @@ public class DMUserInformationProcessor extends DataMigrationProcessorService { ...@@ -40,6 +40,7 @@ public class DMUserInformationProcessor extends DataMigrationProcessorService {
public static final String DM_FAILED_USER = "dm:failed:uid"; public static final String DM_FAILED_USER = "dm:failed:uid";
public static final String DM_EXISTS_USER = "dm:exists:uid"; public static final String DM_EXISTS_USER = "dm:exists:uid";
public static Map<String, AdamTagVo> sexMap = new HashMap<>(); public static Map<String, AdamTagVo> sexMap = new HashMap<>();
static { static {
sexMap.put("SECRET", AdamTagVo.getNew().setVal("MS00").setDesc("保密")); sexMap.put("SECRET", AdamTagVo.getNew().setVal("MS00").setDesc("保密"));
sexMap.put("MALE", AdamTagVo.getNew().setVal("MS01").setDesc("男性")); sexMap.put("MALE", AdamTagVo.getNew().setVal("MS01").setDesc("男性"));
...@@ -50,7 +51,8 @@ public class DMUserInformationProcessor extends DataMigrationProcessorService { ...@@ -50,7 +51,8 @@ public class DMUserInformationProcessor extends DataMigrationProcessorService {
@Override @Override
protected void dataProcessing() { protected void dataProcessing() {
HttpServletRequest request = ServletUtils.getRequest(); HttpServletRequest request = ServletUtils.getRequest();
String lS = request.getParameter(DM_LS), flg = StringUtils.defaultIfBlank(request.getParameter(DM_FLG), ""); String lS = request.getParameter(DM_LS), incrDt = request.getParameter(DM_INCR_DT),
flg = StringUtils.defaultIfBlank(request.getParameter(DM_FLG), "");
if (StringUtils.isNotEmpty(request.getParameter(DM_FLUSH))) { if (StringUtils.isNotEmpty(request.getParameter(DM_FLUSH))) {
log.info("DM.flush.user:{}", userMapper.executeForDM("adam_user")); log.info("DM.flush.user:{}", userMapper.executeForDM("adam_user"));
log.info("DM.flush.user_info:{}", userMapper.executeForDM("adam_user_info")); log.info("DM.flush.user_info:{}", userMapper.executeForDM("adam_user_info"));
...@@ -58,7 +60,7 @@ public class DMUserInformationProcessor extends DataMigrationProcessorService { ...@@ -58,7 +60,7 @@ public class DMUserInformationProcessor extends DataMigrationProcessorService {
log.info("DM.flush.AdamUserInfoVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").exists(true)), AdamUserInfoVo.class.getSimpleName()).getDeletedCount()); log.info("DM.flush.AdamUserInfoVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").exists(true)), AdamUserInfoVo.class.getSimpleName()).getDeletedCount());
} }
String sqlCount = "select count(1) from users where delete_tag in (0,1) and length(mobile)=11 "; String sqlCount = "select count(1) from users where delete_tag in (0,1) and length(mobile)=11 and created_at" + (StringUtils.isBlank(incrDt) ? "<" : ">=") + "curdate()";
String field = "uid,mobile,username,sex,birthday,area,sign,face_url,back_url,created_at,updated_at,is_complete,rong_cloud_token,rong_cloud_tag"; String field = "uid,mobile,username,sex,birthday,area,sign,face_url,back_url,created_at,updated_at,is_complete,rong_cloud_token,rong_cloud_tag";
// 1-uid,2-mobile,3-username,4-sex,5-birthday,6-area,7-sign,8-face_url,9-back_url,10-created_at,11-updated_at,12-is_complete,13-rong_cloud_token,14-rong_cloud_tag // 1-uid,2-mobile,3-username,4-sex,5-birthday,6-area,7-sign,8-face_url,9-back_url,10-created_at,11-updated_at,12-is_complete,13-rong_cloud_token,14-rong_cloud_tag
String sql = sqlCount.replace("count(1)", field); String sql = sqlCount.replace("count(1)", field);
...@@ -66,13 +68,13 @@ public class DMUserInformationProcessor extends DataMigrationProcessorService { ...@@ -66,13 +68,13 @@ public class DMUserInformationProcessor extends DataMigrationProcessorService {
sqlCount += "and mobile in ('13753596360','17701223310','18518013286','18548596019','15901093014','17382319723','13631444097','13671737873')"; sqlCount += "and mobile in ('13753596360','17701223310','18518013286','18548596019','15901093014','17382319723','13631444097','13671737873')";
sql += " and mobile in ('13753596360','17701223310','18518013286','18548596019','15901093014','17382319723','13631444097','13671737873')"; sql += " and mobile in ('13753596360','17701223310','18518013286','18548596019','15901093014','17382319723','13631444097','13671737873')";
} }
sql += " limit ?,?"; sql += " order by uid limit ?,?";
statement = connection.prepareStatement(sqlCount); statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
row.first(); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct; int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start", pl, num, pSize, tl); log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
while (ct > 0) { while (ct > 0) {
statement = connection.prepareStatement(sql); statement = connection.prepareStatement(sql);
......
...@@ -29,14 +29,15 @@ public class DMUserMemberProcessor extends DataMigrationProcessorService { ...@@ -29,14 +29,15 @@ public class DMUserMemberProcessor extends DataMigrationProcessorService {
@Override @Override
protected void dataProcessing() { protected void dataProcessing() {
HttpServletRequest request = ServletUtils.getRequest(); HttpServletRequest request = ServletUtils.getRequest();
String lS = request.getParameter(DM_LS), flg = StringUtils.defaultIfBlank(request.getParameter(DM_FLG), ""); String lS = request.getParameter(DM_LS), incrDt = request.getParameter(DM_INCR_DT),
flg = StringUtils.defaultIfBlank(request.getParameter(DM_FLG), "");
if (StringUtils.isNotEmpty(request.getParameter(DM_FLUSH))) { if (StringUtils.isNotEmpty(request.getParameter(DM_FLUSH))) {
log.info("DM.flush.user_member:{}", userMapper.executeForDM("adam_user_member")); log.info("DM.flush.user_member:{}", userMapper.executeForDM("adam_user_member"));
log.info("DM.flush.AdamUserMemberVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").exists(true)), AdamUserMemberVo.class.getSimpleName()).getDeletedCount()); log.info("DM.flush.AdamUserMemberVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").exists(true)), AdamUserMemberVo.class.getSimpleName()).getDeletedCount());
} }
String sqlCount = "select count(1) from user_cards where 1=1"; String sqlCount = "select count(1) from user_cards where created_at" + (StringUtils.isBlank(incrDt) ? "<" : ">=") + "curdate()";
String field = "uid,card_id,card_number,`status`,expired_at,created_at,updated_at"; String field = "uid,card_id,card_number,`status`,expired_at,created_at,updated_at";
// 1-uid,2-card_id,3-card_number,4-status,5-expired_at,6-created_at,7-updated_at // 1-uid,2-card_id,3-card_number,4-status,5-expired_at,6-created_at,7-updated_at
String sql = sqlCount.replace("count(1)", field); String sql = sqlCount.replace("count(1)", field);
...@@ -44,13 +45,13 @@ public class DMUserMemberProcessor extends DataMigrationProcessorService { ...@@ -44,13 +45,13 @@ public class DMUserMemberProcessor extends DataMigrationProcessorService {
sqlCount += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')"; sqlCount += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
sql += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')"; sql += " and uid in ('2026628','544327','1997335','773650','809406','1167451','423063','677171')";
} }
sql += " limit ?,?"; sql += " order by id limit ?,?";
statement = connection.prepareStatement(sqlCount); statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
row.first(); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct; int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start", pl, num, pSize, tl); log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
while (ct > 0) { while (ct > 0) {
statement = connection.prepareStatement(sql); statement = connection.prepareStatement(sql);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment