记得上下班打卡 | git大法好，push需谨慎

Skip to content

L
liquidnet-bus-v1
Commit a1485b47 authored by 张国柄's avatar 张国柄
Browse files
Options

opt dm;

parent c52c780b
Hide whitespace changes
Inline Side-by-side
Showing with 829 additions and 632 deletions
liquidnet-bus-config/liquidnet-config/liquidnet-service-adam.yml
View file @ a1485b47
...@@ -118,21 +118,14 @@ spring: ...@@ -118,21 +118,14 @@ spring:
url: jdbc:mysql://${liquidnet.mysql.urlHostAndPort}/${liquidnet.mysql.database-name}?serverTimezone=Asia/Shanghai&characterEncoding=utf-8&useSSL=false url: jdbc:mysql://${liquidnet.mysql.urlHostAndPort}/${liquidnet.mysql.database-name}?serverTimezone=Asia/Shanghai&characterEncoding=utf-8&useSSL=false
username: ${liquidnet.mysql.username} username: ${liquidnet.mysql.username}
password: ${liquidnet.mysql.password} password: ${liquidnet.mysql.password}
# type: org.apache.tomcat.jdbc.pool.DataSource # type: org.apache.tomcat.jdbc.pool.DataSource
driver-class-name: com.mysql.cj.jdbc.Driver driver-class-name: com.mysql.cj.jdbc.Driver
filters: stat hikari:
maxActive: 200 maximum-pool-size: 500
initialSize: 1 # 迁移数据暂设10分钟
maxWait: 60000 connection-timeout: 600000
minIdle: 1 minimum-idle: 10
timeBetweenEvictionRunsMillis: 60000 connection-test-query: SELECT 1
minEvictableIdleTimeMillis: 300000
validationQuery: select 'x'
testWhileIdle: true
testOnBorrow: false
testOnReturn: false
poolPreparedStatements: true
maxOpenPreparedStatements: 20
# ----------------------------------------------------------- # -----------------------------------------------------------
# ----------------------------------------------------------- # -----------------------------------------------------------
......
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/docu/db_ln_adam_initialdata.sql
View file @ a1485b47
...@@ -42,7 +42,7 @@ create table adam_user ...@@ -42,7 +42,7 @@ create table adam_user
( (
mid bigint unsigned auto_increment primary key, mid bigint unsigned auto_increment primary key,
uid varchar(64) not null, uid varchar(64) not null,
mobile varchar(30) not null, mobile varchar(30),
passwd varchar(64), passwd varchar(64),
pay_code varchar(64), pay_code varchar(64),
state tinyint default 0 comment '1-NORMAL,2-INVALID', state tinyint default 0 comment '1-NORMAL,2-INVALID',
......
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/controller/admin/AdamDMAdminController.java
View file @ a1485b47
...@@ -67,6 +67,7 @@ public class AdamDMAdminController extends AdamBaseController { ...@@ -67,6 +67,7 @@ public class AdamDMAdminController extends AdamBaseController {
log.info("==================" + k); log.info("==================" + k);
dmUserInformationProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : ""); dmUserInformationProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : "");
}); });
try { Thread.sleep(100L); } catch (InterruptedException ignored) {}
} }
exec.shutdown(); exec.shutdown();
} else { } else {
...@@ -95,6 +96,7 @@ public class AdamDMAdminController extends AdamBaseController { ...@@ -95,6 +96,7 @@ public class AdamDMAdminController extends AdamBaseController {
log.info("==================" + k); log.info("==================" + k);
dmThirdPartsProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : ""); dmThirdPartsProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : "");
}); });
try { Thread.sleep(100L); } catch (InterruptedException ignored) {}
} }
exec.shutdown(); exec.shutdown();
} else { } else {
...@@ -123,6 +125,7 @@ public class AdamDMAdminController extends AdamBaseController { ...@@ -123,6 +125,7 @@ public class AdamDMAdminController extends AdamBaseController {
log.info("==================" + k); log.info("==================" + k);
dmRealNameProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : ""); dmRealNameProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : "");
}); });
try { Thread.sleep(100L); } catch (InterruptedException ignored) {}
} }
exec.shutdown(); exec.shutdown();
} else { } else {
...@@ -151,6 +154,7 @@ public class AdamDMAdminController extends AdamBaseController { ...@@ -151,6 +154,7 @@ public class AdamDMAdminController extends AdamBaseController {
log.info("==================" + k); log.info("==================" + k);
dmCollectionProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : ""); dmCollectionProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : "");
}); });
try { Thread.sleep(100L); } catch (InterruptedException ignored) {}
} }
exec.shutdown(); exec.shutdown();
} else { } else {
...@@ -179,6 +183,7 @@ public class AdamDMAdminController extends AdamBaseController { ...@@ -179,6 +183,7 @@ public class AdamDMAdminController extends AdamBaseController {
log.info("==================" + k); log.info("==================" + k);
dmEntersProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : ""); dmEntersProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : "");
}); });
try { Thread.sleep(100L); } catch (InterruptedException ignored) {}
} }
exec.shutdown(); exec.shutdown();
} else { } else {
...@@ -207,6 +212,7 @@ public class AdamDMAdminController extends AdamBaseController { ...@@ -207,6 +212,7 @@ public class AdamDMAdminController extends AdamBaseController {
log.info("==================" + k); log.info("==================" + k);
dmAddressesProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : ""); dmAddressesProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : "");
}); });
try { Thread.sleep(100L); } catch (InterruptedException ignored) {}
} }
exec.shutdown(); exec.shutdown();
} else { } else {
...@@ -235,6 +241,7 @@ public class AdamDMAdminController extends AdamBaseController { ...@@ -235,6 +241,7 @@ public class AdamDMAdminController extends AdamBaseController {
log.info("==================" + k); log.info("==================" + k);
dmUserMemberProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : ""); dmUserMemberProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : "");
}); });
try { Thread.sleep(100L); } catch (InterruptedException ignored) {}
} }
exec.shutdown(); exec.shutdown();
} else { } else {
......
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMAddressesProcessor.java
View file @ a1485b47
...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired; ...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
...@@ -47,61 +47,82 @@ public class DMAddressesProcessor extends DataMigrationProcessorService { ...@@ -47,61 +47,82 @@ public class DMAddressesProcessor extends DataMigrationProcessorService {
} }
sql += " order by id limit ?,?"; sql += " order by id limit ?,?";
Class.forName(DB_DRIVER); Connection connection = null;
Connection connection = DriverManager.getConnection(DB_URL + DB_NAME_MALL, DB_USER, DB_PWD); PreparedStatement statement = null;
ResultSetImpl row = null;
PreparedStatement statement = connection.prepareStatement(sqlCount); try {
ResultSetImpl row = (ResultSetImpl) statement.executeQuery(); Class.forName(DB_DRIVER);
row.first(); connection = DriverManager.getConnection(DB_URL + DB_NAME_MALL, DB_USER, DB_PWD);
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct; statement = connection.prepareStatement(sqlCount);
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
while (ct > 0) {
statement = connection.prepareStatement(sql);
int ls = pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} - begin", ls, pSize); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
List<AdamAddresses> addressesList = new ArrayList<>(); row.close();
List<AdamAddressesVo> vos = new ArrayList<>(); statement.close();
while (row.next()) { while (ct > 0) {
String uid = row.getString(2); statement = connection.prepareStatement(sql);
int ls = pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} - begin", ls, pSize);
if (!reUidList.contains(uid)) { List<AdamAddresses> addressesList = new ArrayList<>();
AdamAddresses addresses = new AdamAddresses(); List<AdamAddressesVo> vos = new ArrayList<>();
addresses.setAddressesId(row.getString(1)); while (row.next()) {
addresses.setUid(uid); String uid = row.getString(2);
addresses.setName(row.getString(3)); if (!reUidList.contains(uid)) {
addresses.setPhone(row.getString(4)); AdamAddresses addresses = new AdamAddresses();
addresses.setProvince(row.getString(5)); addresses.setAddressesId(row.getString(1));
addresses.setCity(row.getString(6)); addresses.setUid(uid);
addresses.setCounty(row.getString(7)); addresses.setName(row.getString(3));
addresses.setAddress(row.getString(8)); addresses.setPhone(row.getString(4));
addresses.setIsDefault(row.getBoolean(9)); addresses.setProvince(row.getString(5));
addresses.setState(1); addresses.setCity(row.getString(6));
addresses.setCreatedAt(row.getLocalDateTime(10)); addresses.setCounty(row.getString(7));
addresses.setAddress(row.getString(8));
addresses.setIsDefault(row.getBoolean(9));
addresses.setState(1);
addresses.setCreatedAt(row.getLocalDateTime(10));
AdamAddressesVo vo = AdamAddressesVo.getNew().copy(addresses); AdamAddressesVo vo = AdamAddressesVo.getNew().copy(addresses);
addressesList.add(addresses); addressesList.add(addresses);
vos.add(vo); vos.add(vo);
}
if (addressesList.size() == 500) {
tl += addressesList.size();
mongoTemplate.insert(vos, AdamAddressesVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamAddressesService.saveBatch(addressesList));
vos.clear();
addressesList.clear();
}
} }
} row.close();
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, addressesList.size()); statement.close();
if (!CollectionUtils.isEmpty(addressesList)) {
tl += addressesList.size(); num++;
mongoTemplate.insert(vos, AdamAddressesVo.class.getSimpleName()); ct -= pSize;
log.info("DM.execute.limit.result:{}", adamAddressesService.saveBatch(addressesList));
}
num++; log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, tl);
ct -= pSize; }
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
} catch (Exception e) {
log.error("ex:getConnection,dG/dg:{}/{},msg:{}", dG, dg, e.getMessage());
}
try {
if (null != row) row.close();
} catch (SQLException ignored) {
}
try {
if (null != statement) statement.close();
} catch (SQLException ignored) {
}
try {
if (null != connection) connection.close();
} catch (SQLException ignored) {
} }
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
row.close();
statement.close();
connection.close();
} }
} }
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMCollectionProcessor.java
View file @ a1485b47
...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired; ...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
...@@ -35,7 +35,7 @@ public class DMCollectionProcessor extends DataMigrationProcessorService { ...@@ -35,7 +35,7 @@ public class DMCollectionProcessor extends DataMigrationProcessorService {
log.info("DM.flush.AdamCollectVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").exists(true)), AdamCollectVo.class.getSimpleName()).getDeletedCount()); log.info("DM.flush.AdamCollectVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").exists(true)), AdamCollectVo.class.getSimpleName()).getDeletedCount());
} }
String sqlCount = "select count(1) from user_collections where type='TICKET' and status=1 and content_id>=5722 and created_at"+(StringUtils.isBlank(incrDt) ? "<" : ">=")+"curdate()"; String sqlCount = "select count(1) from user_collections where type='TICKET' and status=1 and content_id>=5722 and created_at" + (StringUtils.isBlank(incrDt) ? "<" : ">=") + "curdate()";
if (null != dg) { if (null != dg) {
sqlCount = sqlCount + " and id%" + dG + "=" + dg; sqlCount = sqlCount + " and id%" + dG + "=" + dg;
} }
...@@ -48,53 +48,75 @@ public class DMCollectionProcessor extends DataMigrationProcessorService { ...@@ -48,53 +48,75 @@ public class DMCollectionProcessor extends DataMigrationProcessorService {
} }
sql += " order by id limit ?,?"; sql += " order by id limit ?,?";
Class.forName(DB_DRIVER); Connection connection = null;
Connection connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD); PreparedStatement statement = null;
ResultSetImpl row = null;
PreparedStatement statement = connection.prepareStatement(sqlCount); try {
ResultSetImpl row = (ResultSetImpl) statement.executeQuery(); Class.forName(DB_DRIVER);
row.first(); connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD);
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct; statement = connection.prepareStatement(sqlCount);
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
while (ct > 0) {
statement = connection.prepareStatement(sql);
int ls = pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} - begin", ls, pSize); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
row.close();
statement.close();
while (ct > 0) {
statement = connection.prepareStatement(sql);
int ls = pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} - begin", ls, pSize);
List<AdamCollection> collectionList = new ArrayList<>(); List<AdamCollection> collectionList = new ArrayList<>();
List<AdamCollectVo> vos = new ArrayList<>(); List<AdamCollectVo> vos = new ArrayList<>();
while (row.next()) { while (row.next()) {
String uid = row.getString(1); String uid = row.getString(1);
if (!reUidList.contains(uid)) { if (!reUidList.contains(uid)) {
AdamCollection collection = new AdamCollection(); AdamCollection collection = new AdamCollection();
collection.setUid(uid); collection.setUid(uid);
collection.setContentId(row.getString(2)); collection.setContentId(row.getString(2));
collection.setType(row.getString(3)); collection.setType(row.getString(3));
collection.setState(row.getInt(4)); collection.setState(row.getInt(4));
collection.setCreatedAt(row.getLocalDateTime(5)); collection.setCreatedAt(row.getLocalDateTime(5));
collection.setUpdatedAt(row.getLocalDateTime(6)); collection.setUpdatedAt(row.getLocalDateTime(6));
collectionList.add(collection); collectionList.add(collection);
vos.add(AdamCollectVo.getNew().copy(collection)); vos.add(AdamCollectVo.getNew().copy(collection));
}
if (collectionList.size() == 500) {
tl += collectionList.size();
mongoTemplate.insert(vos, AdamCollectVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamCollectionService.saveBatch(collectionList));
vos.clear();
collectionList.clear();
}
} }
} row.close();
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, collectionList.size()); statement.close();
if (!CollectionUtils.isEmpty(collectionList)) {
tl += collectionList.size();
mongoTemplate.insert(vos, AdamCollectVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamCollectionService.saveBatch(collectionList));
}
num++; num++;
ct -= pSize; ct -= pSize;
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, tl);
}
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
} catch (Exception e) {
log.error("ex:getConnection,dG/dg:{}/{},msg:{}", dG, dg, e.getMessage());
}
try {
if (null != row) row.close();
} catch (SQLException ignored) {
}
try {
if (null != statement) statement.close();
} catch (SQLException ignored) {
}
try {
if (null != connection) connection.close();
} catch (SQLException ignored) {
} }
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
row.close();
statement.close();
connection.close();
} }
} }
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMEntersProcessor.java
View file @ a1485b47
...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired; ...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
...@@ -48,59 +48,80 @@ public class DMEntersProcessor extends DataMigrationProcessorService { ...@@ -48,59 +48,80 @@ public class DMEntersProcessor extends DataMigrationProcessorService {
} }
sql += " order by id limit ?,?"; sql += " order by id limit ?,?";
Class.forName(DB_DRIVER); Connection connection = null;
Connection connection = DriverManager.getConnection(DB_URL + DB_NAME_MALL, DB_USER, DB_PWD); PreparedStatement statement = null;
ResultSetImpl row = null;
PreparedStatement statement = connection.prepareStatement(sqlCount); try {
ResultSetImpl row = (ResultSetImpl) statement.executeQuery(); Class.forName(DB_DRIVER);
row.first(); connection = DriverManager.getConnection(DB_URL + DB_NAME_MALL, DB_USER, DB_PWD);
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct; statement = connection.prepareStatement(sqlCount);
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
while (ct > 0) {
statement = connection.prepareStatement(sql);
int ls = pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} - begin", ls, pSize); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
List<AdamEnters> entersList = new ArrayList<>(); row.close();
List<AdamEntersVo> vos = new ArrayList<>(); statement.close();
while (row.next()) { while (ct > 0) {
String uid = row.getString(2); statement = connection.prepareStatement(sql);
int ls = pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} - begin", ls, pSize);
if (!reUidList.contains(uid)) { List<AdamEnters> entersList = new ArrayList<>();
AdamEnters enters = new AdamEnters(); List<AdamEntersVo> vos = new ArrayList<>();
enters.setEntersId(row.getString(1)); while (row.next()) {
enters.setUid(uid); String uid = row.getString(2);
enters.setType(row.getInt(3)); if (!reUidList.contains(uid)) {
enters.setName(row.getString(4)); AdamEnters enters = new AdamEnters();
enters.setMobile(row.getString(5)); enters.setEntersId(row.getString(1));
enters.setIdCard(row.getString(6)); enters.setUid(uid);
enters.setIsDefault(row.getBoolean(7)); enters.setType(row.getInt(3));
enters.setState(1); enters.setName(row.getString(4));
enters.setCreatedAt(row.getLocalDateTime(8)); enters.setMobile(row.getString(5));
enters.setIdCard(row.getString(6));
enters.setIsDefault(row.getBoolean(7));
enters.setState(1);
enters.setCreatedAt(row.getLocalDateTime(8));
AdamEntersVo vo = AdamEntersVo.getNew().copy(enters); AdamEntersVo vo = AdamEntersVo.getNew().copy(enters);
entersList.add(enters); entersList.add(enters);
vos.add(vo); vos.add(vo);
}
if (entersList.size() == 500) {
tl += entersList.size();
mongoTemplate.insert(vos, AdamEntersVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamEntersService.saveBatch(entersList));
vos.clear();
entersList.clear();
}
} }
} row.close();
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, entersList.size()); statement.close();
if (!CollectionUtils.isEmpty(entersList)) {
tl += entersList.size(); num++;
mongoTemplate.insert(vos, AdamEntersVo.class.getSimpleName()); ct -= pSize;
log.info("DM.execute.limit.result:{}", adamEntersService.saveBatch(entersList));
}
num++; log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, tl);
ct -= pSize; }
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
} catch (Exception e) {
log.error("ex:getConnection,dG/dg:{}/{},msg:{}", dG, dg, e.getMessage());
}
try {
if (null != row) row.close();
} catch (SQLException ignored) {
}
try {
if (null != statement) statement.close();
} catch (SQLException ignored) {
}
try {
if (null != connection) connection.close();
} catch (SQLException ignored) {
} }
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
row.close();
statement.close();
connection.close();
} }
} }
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMMemberCodeProcessor.java
View file @ a1485b47
...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired; ...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
...@@ -29,154 +29,155 @@ public class DMMemberCodeProcessor extends DataMigrationProcessorService { ...@@ -29,154 +29,155 @@ public class DMMemberCodeProcessor extends DataMigrationProcessorService {
IAdamMemberCodeService adamMemberCodeService; IAdamMemberCodeService adamMemberCodeService;
private static Map<String, String> mMap = new HashMap<>(); private static Map<String, String> mMap = new HashMap<>();
static { static {
mMap.put("78","2159902978359319814V"); mMap.put("78", "2159902978359319814V");
mMap.put("93","2159903004882502855V"); mMap.put("93", "2159903004882502855V");
mMap.put("128","2159903083063113534V"); mMap.put("128", "2159903083063113534V");
mMap.put("171","2159903164279244032V"); mMap.put("171", "2159903164279244032V");
mMap.put("177","2159903178213564580V"); mMap.put("177", "2159903178213564580V");
mMap.put("201","2159903231304794843V"); mMap.put("201", "2159903231304794843V");
mMap.put("224","2159903286921267622V"); mMap.put("224", "2159903286921267622V");
mMap.put("278","2159903315535368620V"); mMap.put("278", "2159903315535368620V");
mMap.put("308","2159903325648681321V"); mMap.put("308", "2159903325648681321V");
mMap.put("371","2159903353285533024V"); mMap.put("371", "2159903353285533024V");
mMap.put("406","2159903365042471904V"); mMap.put("406", "2159903365042471904V");
mMap.put("448","2159903380066742978V"); mMap.put("448", "2159903380066742978V");
mMap.put("470","2159903389019474764V"); mMap.put("470", "2159903389019474764V");
mMap.put("514","2159903402623760508V"); mMap.put("514", "2159903402623760508V");
mMap.put("564","2159903420100480044V"); mMap.put("564", "2159903420100480044V");
mMap.put("566","2159903420815652272V"); mMap.put("566", "2159903420815652272V");
mMap.put("572","2159903425097220662V"); mMap.put("572", "2159903425097220662V");
mMap.put("584","2159903431725878110V"); mMap.put("584", "2159903431725878110V");
mMap.put("621","2159903445678115851V"); mMap.put("621", "2159903445678115851V");
mMap.put("625","2159903448130804563V"); mMap.put("625", "2159903448130804563V");
mMap.put("629","2159903449512021348V"); mMap.put("629", "2159903449512021348V");
mMap.put("634","2159903451797722527V"); mMap.put("634", "2159903451797722527V");
mMap.put("641","2159903456947872759V"); mMap.put("641", "2159903456947872759V");
mMap.put("649","2159903462764769342V"); mMap.put("649", "2159903462764769342V");
mMap.put("663","2159903470899755168V"); mMap.put("663", "2159903470899755168V");
mMap.put("670","2159903474711853726V"); mMap.put("670", "2159903474711853726V");
mMap.put("671","2159903474789135882V"); mMap.put("671", "2159903474789135882V");
mMap.put("692","2159903489229795967V"); mMap.put("692", "2159903489229795967V");
mMap.put("694","2159903491297244057V"); mMap.put("694", "2159903491297244057V");
mMap.put("724","2159903503090588386V"); mMap.put("724", "2159903503090588386V");
mMap.put("744","2159903517757370778V"); mMap.put("744", "2159903517757370778V");
mMap.put("753","2159903521221866688V"); mMap.put("753", "2159903521221866688V");
mMap.put("767","2159903529347024019V"); mMap.put("767", "2159903529347024019V");
mMap.put("812","2159903555573538568V"); mMap.put("812", "2159903555573538568V");
mMap.put("821","2159903560265023540V"); mMap.put("821", "2159903560265023540V");
mMap.put("831","2159903566740588324V"); mMap.put("831", "2159903566740588324V");
mMap.put("847","2159903574058019392V"); mMap.put("847", "2159903574058019392V");
mMap.put("880","2159903599623697117V"); mMap.put("880", "2159903599623697117V");
mMap.put("884","2159903605349638303V"); mMap.put("884", "2159903605349638303V");
mMap.put("893","2159903612326372870V"); mMap.put("893", "2159903612326372870V");
mMap.put("897","2159903630571607844V"); mMap.put("897", "2159903630571607844V");
mMap.put("899","2159903631906318964V"); mMap.put("899", "2159903631906318964V");
mMap.put("915","2159903663070164126V"); mMap.put("915", "2159903663070164126V");
mMap.put("925","2159903708638226752V"); mMap.put("925", "2159903708638226752V");
mMap.put("926","2159903709187446675V"); mMap.put("926", "2159903709187446675V");
mMap.put("929","2159903725057215138V"); mMap.put("929", "2159903725057215138V");
mMap.put("932","2159903733267427426V"); mMap.put("932", "2159903733267427426V");
mMap.put("934","2159903746991129377V"); mMap.put("934", "2159903746991129377V");
mMap.put("945","2159903899372919752V"); mMap.put("945", "2159903899372919752V");
mMap.put("946","2159903944921229059V"); mMap.put("946", "2159903944921229059V");
mMap.put("947","2159903973383580976V"); mMap.put("947", "2159903973383580976V");
mMap.put("948","2159904743264389337V"); mMap.put("948", "2159904743264389337V");
mMap.put("949","2159904881753208429V"); mMap.put("949", "2159904881753208429V");
mMap.put("950","2159905009806735836V"); mMap.put("950", "2159905009806735836V");
mMap.put("951","2159905284996580808V"); mMap.put("951", "2159905284996580808V");
mMap.put("952","2159905286677634113V"); mMap.put("952", "2159905286677634113V");
mMap.put("953","2159905455234006614V"); mMap.put("953", "2159905455234006614V");
mMap.put("954","2159905499349116774V"); mMap.put("954", "2159905499349116774V");
mMap.put("955","2159905536256825603V"); mMap.put("955", "2159905536256825603V");
mMap.put("956","2159906003655765269V"); mMap.put("956", "2159906003655765269V");
mMap.put("957","2159909717834813970V"); mMap.put("957", "2159909717834813970V");
mMap.put("958","2159910092253472736V"); mMap.put("958", "2159910092253472736V");
mMap.put("959","2159910193014350228V"); mMap.put("959", "2159910193014350228V");
mMap.put("960","2159910527370977102V"); mMap.put("960", "2159910527370977102V");
mMap.put("961","2159910531884733969V"); mMap.put("961", "2159910531884733969V");
mMap.put("962","2159910679489574768V"); mMap.put("962", "2159910679489574768V");
mMap.put("963","2159910910443346816V"); mMap.put("963", "2159910910443346816V");
mMap.put("964","2159912826218304603V"); mMap.put("964", "2159912826218304603V");
mMap.put("965","1599558421V"); mMap.put("965", "1599558421V");
mMap.put("966","1599558422V"); mMap.put("966", "1599558422V");
mMap.put("967","1599558423V"); mMap.put("967", "1599558423V");
mMap.put("968","1599558424V"); mMap.put("968", "1599558424V");
mMap.put("969","1599558425V"); mMap.put("969", "1599558425V");
mMap.put("970","1599558426V"); mMap.put("970", "1599558426V");
mMap.put("971","1599558427V"); mMap.put("971", "1599558427V");
mMap.put("972","1599558428V"); mMap.put("972", "1599558428V");
mMap.put("973","1599558429V"); mMap.put("973", "1599558429V");
mMap.put("974","1599558431V"); mMap.put("974", "1599558431V");
mMap.put("975","2159956316098104568V"); mMap.put("975", "2159956316098104568V");
mMap.put("976","2159956355872033001V"); mMap.put("976", "2159956355872033001V");
mMap.put("977","2159956860062026155V"); mMap.put("977", "2159956860062026155V");
mMap.put("978","2159957584538985726V"); mMap.put("978", "2159957584538985726V");
mMap.put("979","2159957623536849859V"); mMap.put("979", "2159957623536849859V");
mMap.put("980","2159957731800533965V"); mMap.put("980", "2159957731800533965V");
mMap.put("981","2159957840680405904V"); mMap.put("981", "2159957840680405904V");
mMap.put("982","2159957929849237090V"); mMap.put("982", "2159957929849237090V");
mMap.put("983","2159962320514458284V"); mMap.put("983", "2159962320514458284V");
mMap.put("984","1599631361V"); mMap.put("984", "1599631361V");
mMap.put("985","1599631362V"); mMap.put("985", "1599631362V");
mMap.put("986","1599631363V"); mMap.put("986", "1599631363V");
mMap.put("987","1599631364V"); mMap.put("987", "1599631364V");
mMap.put("988","1599631365V"); mMap.put("988", "1599631365V");
mMap.put("989","1599631366V"); mMap.put("989", "1599631366V");
mMap.put("990","1599631367V"); mMap.put("990", "1599631367V");
mMap.put("991","1599631368V"); mMap.put("991", "1599631368V");
mMap.put("992","1599631369V"); mMap.put("992", "1599631369V");
mMap.put("993","1599631370V"); mMap.put("993", "1599631370V");
mMap.put("994","2159963377631095182V"); mMap.put("994", "2159963377631095182V");
mMap.put("995","2159963692908504280V"); mMap.put("995", "2159963692908504280V");
mMap.put("996","2159963810907987285V"); mMap.put("996", "2159963810907987285V");
mMap.put("997","2159965666281774447V"); mMap.put("997", "2159965666281774447V");
mMap.put("1000","2159966779640863040V"); mMap.put("1000", "2159966779640863040V");
mMap.put("1062","2159981484289308634V"); mMap.put("1062", "2159981484289308634V");
mMap.put("1104","2159981487011831223V"); mMap.put("1104", "2159981487011831223V");
mMap.put("1114","2159981487541782867V"); mMap.put("1114", "2159981487541782867V");
mMap.put("1198","2159981493699630131V"); mMap.put("1198", "2159981493699630131V");
mMap.put("1200","2159981493787551500V"); mMap.put("1200", "2159981493787551500V");
mMap.put("1235","2159981500360161310V"); mMap.put("1235", "2159981500360161310V");
mMap.put("1262","2159981509978829277V"); mMap.put("1262", "2159981509978829277V");
mMap.put("1287","2159981524026090875V"); mMap.put("1287", "2159981524026090875V");
mMap.put("1304","2159981536177567220V"); mMap.put("1304", "2159981536177567220V");
mMap.put("1310","2159981545302191346V"); mMap.put("1310", "2159981545302191346V");
mMap.put("1319","2159981554253166294V"); mMap.put("1319", "2159981554253166294V");
mMap.put("1325","2159981560137685485V"); mMap.put("1325", "2159981560137685485V");
mMap.put("1329","2159981563803933569V"); mMap.put("1329", "2159981563803933569V");
mMap.put("1335","2159981573925532296V"); mMap.put("1335", "2159981573925532296V");
mMap.put("1390","2159981688651814636V"); mMap.put("1390", "2159981688651814636V");
mMap.put("1391","2159981690863703353V"); mMap.put("1391", "2159981690863703353V");
mMap.put("1428","2159981809289823292V"); mMap.put("1428", "2159981809289823292V");
mMap.put("1430","2159981818758016923V"); mMap.put("1430", "2159981818758016923V");
mMap.put("1458","2159981953122687002V"); mMap.put("1458", "2159981953122687002V");
mMap.put("1494","2159982351398865567V"); mMap.put("1494", "2159982351398865567V");
mMap.put("1495","2159983504307869358V"); mMap.put("1495", "2159983504307869358V");
mMap.put("1496","2159986673135905440V"); mMap.put("1496", "2159986673135905440V");
mMap.put("1497","2159996274042898483V"); mMap.put("1497", "2159996274042898483V");
mMap.put("1498","2160018496723049508V"); mMap.put("1498", "2160018496723049508V");
mMap.put("1499","2160044520193237286V"); mMap.put("1499", "2160044520193237286V");
mMap.put("1500","2160087799997486838V"); mMap.put("1500", "2160087799997486838V");
mMap.put("2230","2160301409593379808V"); mMap.put("2230", "2160301409593379808V");
mMap.put("2231","2160306912844638435V"); mMap.put("2231", "2160306912844638435V");
mMap.put("2232","2160307407349758361V"); mMap.put("2232", "2160307407349758361V");
mMap.put("2233","2160307601583764655V"); mMap.put("2233", "2160307601583764655V");
mMap.put("2234","2160307779536228677V"); mMap.put("2234", "2160307779536228677V");
mMap.put("2492","159963137111V"); mMap.put("2492", "159963137111V");
mMap.put("2510","2160327308412550248V"); mMap.put("2510", "2160327308412550248V");
mMap.put("2515","2160327423681020972V"); mMap.put("2515", "2160327423681020972V");
mMap.put("3359","2160447748642267709V"); mMap.put("3359", "2160447748642267709V");
mMap.put("3360","2160464108681210878V"); mMap.put("3360", "2160464108681210878V");
mMap.put("6315","2161440540017128902V"); mMap.put("6315", "2161440540017128902V");
mMap.put("6511","2161535805626574069V"); mMap.put("6511", "2161535805626574069V");
mMap.put("6726","2161571734236003828V"); mMap.put("6726", "2161571734236003828V");
mMap.put("7629","2161675246646728746V"); mMap.put("7629", "2161675246646728746V");
mMap.put("9150","2161891364918440203V"); mMap.put("9150", "2161891364918440203V");
mMap.put("9554","2161900346985860326V"); mMap.put("9554", "2161900346985860326V");
mMap.put("9888","2162037264105291340V"); mMap.put("9888", "2162037264105291340V");
mMap.put("9889","2162037313688676374V"); mMap.put("9889", "2162037313688676374V");
mMap.put("11470","2162202689894532474V"); mMap.put("11470", "2162202689894532474V");
} }
@SneakyThrows @SneakyThrows
...@@ -198,26 +199,31 @@ public class DMMemberCodeProcessor extends DataMigrationProcessorService { ...@@ -198,26 +199,31 @@ public class DMMemberCodeProcessor extends DataMigrationProcessorService {
String sql = sqlCount.replace("count(1)", field); String sql = sqlCount.replace("count(1)", field);
sql += " order by id limit ?,?"; sql += " order by id limit ?,?";
Class.forName(DB_DRIVER); Connection connection = null;
Connection connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD); PreparedStatement statement = null;
ResultSetImpl row = null;
PreparedStatement statement = connection.prepareStatement(sqlCount); try {
ResultSetImpl row = (ResultSetImpl) statement.executeQuery(); Class.forName(DB_DRIVER);
row.first(); connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD);
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct; statement = connection.prepareStatement(sqlCount);
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
while (ct > 0) {
statement = connection.prepareStatement(sql);
int ls = pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} - begin", ls, pSize); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
List<AdamMemberCode> memberCodeList = new ArrayList<>(); row.close();
List<AdamMemberCodeVo> vos = new ArrayList<>(); statement.close();
while (row.next()) { while (ct > 0) {
statement = connection.prepareStatement(sql);
int ls = pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} - begin", ls, pSize);
List<AdamMemberCode> memberCodeList = new ArrayList<>();
List<AdamMemberCodeVo> vos = new ArrayList<>();
while (row.next()) {
// String buyUid = row.getString(8); // String buyUid = row.getString(8);
// String useUid = row.getString(11); // String useUid = row.getString(11);
// if (!reUidList.contains(buyUid) && reUidList.contains(useUid)) { // if (!reUidList.contains(buyUid) && reUidList.contains(useUid)) {
...@@ -240,20 +246,37 @@ public class DMMemberCodeProcessor extends DataMigrationProcessorService { ...@@ -240,20 +246,37 @@ public class DMMemberCodeProcessor extends DataMigrationProcessorService {
memberCodeList.add(memberCode); memberCodeList.add(memberCode);
vos.add(AdamMemberCodeVo.getNew().copy(memberCode)); vos.add(AdamMemberCodeVo.getNew().copy(memberCode));
// } // }
} if (memberCodeList.size() == 500) {
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, memberCodeList.size()); tl += memberCodeList.size();
if (!CollectionUtils.isEmpty(memberCodeList)) { mongoTemplate.insert(vos, AdamMemberCodeVo.class.getSimpleName());
tl += memberCodeList.size(); log.info("DM.execute.limit.result:{}", adamMemberCodeService.saveBatch(memberCodeList));
mongoTemplate.insert(vos, AdamMemberCodeVo.class.getSimpleName()); vos.clear();
log.info("DM.execute.limit.result:{}", adamMemberCodeService.saveBatch(memberCodeList)); memberCodeList.clear();
} }
}
row.close();
statement.close();
num++; num++;
ct -= pSize; ct -= pSize;
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, tl);
}
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
} catch (Exception e) {
log.error("ex:getConnection,dG/dg:{}/{},msg:{}", dG, dg, e.getMessage());
}
try {
if (null != row) row.close();
} catch (SQLException ignored) {
}
try {
if (null != statement) statement.close();
} catch (SQLException ignored) {
}
try {
if (null != connection) connection.close();
} catch (SQLException ignored) {
} }
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
row.close();
statement.close();
connection.close();
} }
} }
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMMemberOrderProcessor.java
View file @ a1485b47
...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired; ...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
...@@ -73,71 +73,93 @@ public class DMMemberOrderProcessor extends DataMigrationProcessorService { ...@@ -73,71 +73,93 @@ public class DMMemberOrderProcessor extends DataMigrationProcessorService {
} }
sql += " order by id limit ?,?"; sql += " order by id limit ?,?";
Class.forName(DB_DRIVER); Connection connection = null;
Connection connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD); PreparedStatement statement = null;
ResultSetImpl row = null;
PreparedStatement statement = connection.prepareStatement(sqlCount); try {
ResultSetImpl row = (ResultSetImpl) statement.executeQuery(); Class.forName(DB_DRIVER);
row.first(); connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD);
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct; statement = connection.prepareStatement(sqlCount);
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
while (ct > 0) {
statement = connection.prepareStatement(sql);
int ls = pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} - begin", ls, pSize); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
List<AdamMemberOrder> memberOrderList = new ArrayList<>(); log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
List<AdamMemberOrderVo> vos = new ArrayList<>();
while (row.next()) { row.close();
String uid = row.getString(2); statement.close();
if (!reUidList.contains(uid)) { while (ct > 0) {
AdamMemberOrder memberOrder = new AdamMemberOrder(); statement = connection.prepareStatement(sql);
memberOrder.setOrderNo(row.getString(1)); int ls = pSize * num;
memberOrder.setUid(uid); statement.setInt(1, ls);
memberOrder.setMode(row.getInt(3)); statement.setInt(2, pSize);
memberOrder.setPrice(row.getBigDecimal(4)); row = (ResultSetImpl) statement.executeQuery();
memberOrder.setPricePaid(row.getBigDecimal(5)); log.info("DM.execute.limit {},{} - begin", ls, pSize);
memberOrder.setMemberName(MEMBER_NAME);
memberOrder.setMemberId(MEMBER_ID); List<AdamMemberOrder> memberOrderList = new ArrayList<>();
memberOrder.setMemberPriceId(MEMBER_ID); List<AdamMemberOrderVo> vos = new ArrayList<>();
memberOrder.setDays(365); while (row.next()) {
memberOrder.setState(row.getInt(6)); String uid = row.getString(2);
memberOrder.setMemberNo(row.getString(7)); if (!reUidList.contains(uid)) {
memberOrder.setBirthday(row.getString(8)); AdamMemberOrder memberOrder = new AdamMemberOrder();
String payType = row.getString(9); memberOrder.setOrderNo(row.getString(1));
memberOrder.setDeviceFrom(dfMap.get(payType)); memberOrder.setUid(uid);
memberOrder.setPayType(ptMap.get(payType)); memberOrder.setMode(row.getInt(3));
memberOrder.setPayNo(row.getString(10)); memberOrder.setPrice(row.getBigDecimal(4));
memberOrder.setPaymentAt(row.getLocalDateTime(11)); memberOrder.setPricePaid(row.getBigDecimal(5));
memberOrder.setCreatedAt(row.getLocalDateTime(12)); memberOrder.setMemberName(MEMBER_NAME);
memberOrder.setUpdatedAt(row.getLocalDateTime(13)); memberOrder.setMemberId(MEMBER_ID);
memberOrder.setClientIp(row.getString(14)); memberOrder.setMemberPriceId(MEMBER_ID);
memberOrder.setSource(row.getString(15)); memberOrder.setDays(365);
memberOrder.setVersion(row.getString(16)); memberOrder.setState(row.getInt(6));
memberOrder.setMemberNo(row.getString(7));
AdamMemberOrderVo vo = AdamMemberOrderVo.getNew().copy(memberOrder); memberOrder.setBirthday(row.getString(8));
String payType = row.getString(9);
memberOrderList.add(memberOrder); memberOrder.setDeviceFrom(dfMap.get(payType));
vos.add(vo); memberOrder.setPayType(ptMap.get(payType));
memberOrder.setPayNo(row.getString(10));
memberOrder.setPaymentAt(row.getLocalDateTime(11));
memberOrder.setCreatedAt(row.getLocalDateTime(12));
memberOrder.setUpdatedAt(row.getLocalDateTime(13));
memberOrder.setClientIp(row.getString(14));
memberOrder.setSource(row.getString(15));
memberOrder.setVersion(row.getString(16));
AdamMemberOrderVo vo = AdamMemberOrderVo.getNew().copy(memberOrder);
memberOrderList.add(memberOrder);
vos.add(vo);
}
if (memberOrderList.size() == 500) {
tl += memberOrderList.size();
mongoTemplate.insert(vos, AdamMemberOrderVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamMemberOrderService.saveBatch(memberOrderList));
vos.clear();
memberOrderList.clear();
}
} }
} row.close();
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, memberOrderList.size()); statement.close();
if (!CollectionUtils.isEmpty(memberOrderList)) {
tl += memberOrderList.size(); num++;
mongoTemplate.insert(vos, AdamMemberOrderVo.class.getSimpleName()); ct -= pSize;
log.info("DM.execute.limit.result:{}", adamMemberOrderService.saveBatch(memberOrderList));
}
num++; log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, tl);
ct -= pSize; }
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
} catch (Exception e) {
log.error("ex:getConnection,dG/dg:{}/{},msg:{}", dG, dg, e.getMessage());
}
try {
if (null != row) row.close();
} catch (SQLException ignored) {
}
try {
if (null != statement) statement.close();
} catch (SQLException ignored) {
}
try {
if (null != connection) connection.close();
} catch (SQLException ignored) {
} }
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
row.close();
statement.close();
connection.close();
} }
} }
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMRealNameProcessor.java
View file @ a1485b47
...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired; ...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
...@@ -48,62 +48,84 @@ public class DMRealNameProcessor extends DataMigrationProcessorService { ...@@ -48,62 +48,84 @@ public class DMRealNameProcessor extends DataMigrationProcessorService {
} }
sql += " order by id limit ?,?"; sql += " order by id limit ?,?";
Class.forName(DB_DRIVER); Connection connection = null;
Connection connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD); PreparedStatement statement = null;
ResultSetImpl row = null;
PreparedStatement statement = connection.prepareStatement(sqlCount); try {
ResultSetImpl row = (ResultSetImpl) statement.executeQuery(); Class.forName(DB_DRIVER);
row.first(); connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD);
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct; statement = connection.prepareStatement(sqlCount);
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
while (ct > 0) {
statement = connection.prepareStatement(sql);
int ls = pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} - begin", ls, pSize); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
row.close();
statement.close();
while (ct > 0) {
statement = connection.prepareStatement(sql);
int ls = pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} - begin", ls, pSize);
List<AdamRealName> realNameList = new ArrayList<>(); List<AdamRealName> realNameList = new ArrayList<>();
List<AdamRealInfoVo> vos = new ArrayList<>(); List<AdamRealInfoVo> vos = new ArrayList<>();
while (row.next()) { while (row.next()) {
String uid = row.getString(2); String uid = row.getString(2);
if (!reUidList.contains(uid)) { if (!reUidList.contains(uid)) {
AdamRealName realName = new AdamRealName(); AdamRealName realName = new AdamRealName();
realName.setRealNameId(row.getString(1)); realName.setRealNameId(row.getString(1));
realName.setUid(uid); realName.setUid(uid);
realName.setType("1"); realName.setType("1");
realName.setName(row.getString(3)); realName.setName(row.getString(3));
realName.setIdCard(row.getString(4)); realName.setIdCard(row.getString(4));
realName.setState(1); realName.setState(1);
realName.setCreatedAt(row.getLocalDateTime(5)); realName.setCreatedAt(row.getLocalDateTime(5));
realName.setUpdatedAt(row.getLocalDateTime(6)); realName.setUpdatedAt(row.getLocalDateTime(6));
AdamRealInfoVo vo = AdamRealInfoVo.getNew(); AdamRealInfoVo vo = AdamRealInfoVo.getNew();
vo.setUid(uid); vo.setUid(uid);
vo.setType(realName.getType()); vo.setType(realName.getType());
vo.setName(realName.getName()); vo.setName(realName.getName());
vo.setIdCard(realName.getIdCard()); vo.setIdCard(realName.getIdCard());
vo.setState(1); vo.setState(1);
realNameList.add(realName); realNameList.add(realName);
vos.add(vo); vos.add(vo);
}
if (realNameList.size() == 500) {
tl += realNameList.size();
mongoTemplate.insert(vos, AdamRealInfoVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamRealNameService.saveBatch(realNameList));
vos.clear();
realNameList.clear();
}
} }
} row.close();
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, realNameList.size()); statement.close();
if (!CollectionUtils.isEmpty(realNameList)) {
tl += realNameList.size();
mongoTemplate.insert(vos, AdamRealInfoVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamRealNameService.saveBatch(realNameList));
}
num++; num++;
ct -= pSize; ct -= pSize;
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, tl);
}
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
} catch (Exception e) {
log.error("ex:getConnection,dG/dg:{}/{},msg:{}", dG, dg, e.getMessage());
}
try {
if (null != row) row.close();
} catch (SQLException ignored) {
}
try {
if (null != statement) statement.close();
} catch (SQLException ignored) {
}
try {
if (null != connection) connection.close();
} catch (SQLException ignored) {
} }
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
row.close();
statement.close();
connection.close();
} }
} }
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMThirdPartsProcessor.java
View file @ a1485b47
...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired; ...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
...@@ -36,7 +36,7 @@ public class DMThirdPartsProcessor extends DataMigrationProcessorService { ...@@ -36,7 +36,7 @@ public class DMThirdPartsProcessor extends DataMigrationProcessorService {
} }
String sqlCount = "select count(1) from (\n" + String sqlCount = "select count(1) from (\n" +
" select row_number() over (partition by uid,type order by created_at desc) rn,id,uid,open_id,avatar,nickname,type,created_at,updated_at from user_third_parts where `status`=1 and delete_tag in (0,1) and open_id<>'' and uid>0 and created_at"+(StringUtils.isBlank(incrDt) ? "<" : ">=")+"curdate()\n" + " select row_number() over (partition by uid,type order by created_at desc) rn,id,uid,open_id,avatar,nickname,type,created_at,updated_at from user_third_parts where `status`=1 and delete_tag in (0,1) and open_id<>'' and uid>0 and created_at" + (StringUtils.isBlank(incrDt) ? "<" : ">=") + "curdate()\n" +
" ) t where t.rn=1"; " ) t where t.rn=1";
if (null != dg) { if (null != dg) {
sqlCount = sqlCount + " and id%" + dG + "=" + dg; sqlCount = sqlCount + " and id%" + dG + "=" + dg;
...@@ -50,65 +50,87 @@ public class DMThirdPartsProcessor extends DataMigrationProcessorService { ...@@ -50,65 +50,87 @@ public class DMThirdPartsProcessor extends DataMigrationProcessorService {
} }
sql += " order by id limit ?,?"; sql += " order by id limit ?,?";
Class.forName(DB_DRIVER); Connection connection = null;
Connection connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD); PreparedStatement statement = null;
ResultSetImpl row = null;
PreparedStatement statement = connection.prepareStatement(sqlCount); try {
ResultSetImpl row = (ResultSetImpl) statement.executeQuery(); Class.forName(DB_DRIVER);
row.first(); connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD);
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct; statement = connection.prepareStatement(sqlCount);
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
while (ct > 0) {
statement = connection.prepareStatement(sql);
int ls = pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} - begin", ls, pSize); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
row.close();
statement.close();
while (ct > 0) {
statement = connection.prepareStatement(sql);
int ls = pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} - begin", ls, pSize);
List<AdamThirdParty> thirdPartyList = new ArrayList<>(); List<AdamThirdParty> thirdPartyList = new ArrayList<>();
List<AdamThirdPartInfoVo> vos = new ArrayList<>(); List<AdamThirdPartInfoVo> vos = new ArrayList<>();
while (row.next()) { while (row.next()) {
String uid = row.getString(1); String uid = row.getString(1);
if (!reUidList.contains(uid)) { if (!reUidList.contains(uid)) {
AdamThirdParty thirdParty = new AdamThirdParty(); AdamThirdParty thirdParty = new AdamThirdParty();
thirdParty.setUid(uid); thirdParty.setUid(uid);
thirdParty.setOpenId(row.getString(2)); thirdParty.setOpenId(row.getString(2));
thirdParty.setAvatar(row.getString(3)); thirdParty.setAvatar(row.getString(3));
thirdParty.setNickname(row.getString(4)); thirdParty.setNickname(row.getString(4));
thirdParty.setPlatform(row.getString(5)); thirdParty.setPlatform(row.getString(5));
thirdParty.setState(1); thirdParty.setState(1);
thirdParty.setCreatedAt(row.getLocalDateTime(6)); thirdParty.setCreatedAt(row.getLocalDateTime(6));
thirdParty.setUpdatedAt(row.getLocalDateTime(7)); thirdParty.setUpdatedAt(row.getLocalDateTime(7));
AdamThirdPartInfoVo vo = AdamThirdPartInfoVo.getNew(); AdamThirdPartInfoVo vo = AdamThirdPartInfoVo.getNew();
vo.setUid(thirdParty.getUid()); vo.setUid(thirdParty.getUid());
vo.setOpenId(thirdParty.getOpenId()); vo.setOpenId(thirdParty.getOpenId());
vo.setNickname(thirdParty.getNickname()); vo.setNickname(thirdParty.getNickname());
vo.setAvatar(thirdParty.getAvatar()); vo.setAvatar(thirdParty.getAvatar());
vo.setPlatform(thirdParty.getPlatform()); vo.setPlatform(thirdParty.getPlatform());
vo.setState(thirdParty.getState()); vo.setState(thirdParty.getState());
vo.setCreatedAt(thirdParty.getCreatedAt()); vo.setCreatedAt(thirdParty.getCreatedAt());
vo.setUpdatedAt(thirdParty.getUpdatedAt()); vo.setUpdatedAt(thirdParty.getUpdatedAt());
thirdPartyList.add(thirdParty); thirdPartyList.add(thirdParty);
vos.add(vo); vos.add(vo);
}
if (thirdPartyList.size() == 500) {
tl += thirdPartyList.size();
mongoTemplate.insert(vos, AdamThirdPartInfoVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamThirdPartyService.saveBatch(thirdPartyList));
vos.clear();
thirdPartyList.clear();
}
} }
} row.close();
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, thirdPartyList.size()); statement.close();
if (!CollectionUtils.isEmpty(thirdPartyList)) {
tl += thirdPartyList.size();
mongoTemplate.insert(vos, AdamThirdPartInfoVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamThirdPartyService.saveBatch(thirdPartyList));
}
num++; num++;
ct -= pSize; ct -= pSize;
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, tl);
}
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
} catch (Exception e) {
log.error("ex:getConnection,dG/dg:{}/{},msg:{}", dG, dg, e.getMessage());
}
try {
if (null != row) row.close();
} catch (SQLException ignored) {
}
try {
if (null != statement) statement.close();
} catch (SQLException ignored) {
}
try {
if (null != connection) connection.close();
} catch (SQLException ignored) {
} }
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
row.close();
statement.close();
connection.close();
} }
} }
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMUserInformationProcessor.java
View file @ a1485b47
...@@ -18,11 +18,11 @@ import org.springframework.beans.factory.annotation.Autowired; ...@@ -18,11 +18,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
...@@ -68,85 +68,107 @@ public class DMUserInformationProcessor extends DataMigrationProcessorService { ...@@ -68,85 +68,107 @@ public class DMUserInformationProcessor extends DataMigrationProcessorService {
} }
sql += " order by uid limit ?,?"; sql += " order by uid limit ?,?";
Class.forName(DB_DRIVER); Connection connection = null;
Connection connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD); PreparedStatement statement = null;
ResultSetImpl row = null;
PreparedStatement statement = connection.prepareStatement(sqlCount); try {
ResultSetImpl row = (ResultSetImpl) statement.executeQuery(); Class.forName(DB_DRIVER);
row.first(); connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD);
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct; statement = connection.prepareStatement(sqlCount);
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
while (ct > 0) {
statement = connection.prepareStatement(sql);
int ls = pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} - begin", ls, pSize); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
List<AdamUser> userList = new ArrayList<>(); log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
List<AdamUserInfo> userInfoList = new ArrayList<>();
List<AdamUserInfoVo> vos = new ArrayList<>(); row.close();
while (row.next()) { statement.close();
AdamTagVo sex = sexMap.get(row.getString(4)); while (ct > 0) {
String uid = row.getString(1); statement = connection.prepareStatement(sql);
if (!reUidList.contains(uid)) { int ls = pSize * num;
AdamUser user = new AdamUser(); statement.setInt(1, ls);
user.setUid(uid); statement.setInt(2, pSize);
user.setMobile(row.getString(2)); row = (ResultSetImpl) statement.executeQuery();
user.setCreatedAt(row.getLocalDateTime(10)); log.info("DM.execute.limit {},{} - begin", ls, pSize);
user.setUpdatedAt(row.getLocalDateTime(11));
user.setState(1); List<AdamUser> userList = new ArrayList<>();
List<AdamUserInfo> userInfoList = new ArrayList<>();
AdamUserInfo userInfo = new AdamUserInfo(); List<AdamUserInfoVo> vos = new ArrayList<>();
userInfo.setUid(user.getUid()); while (row.next()) {
userInfo.setNickname(row.getString(3)); AdamTagVo sex = sexMap.get(row.getString(4));
userInfo.setSex(JsonUtils.toJson(sex)); String uid = row.getString(1);
userInfo.setBirthday(row.getLocalDate(5)); if (!reUidList.contains(uid)) {
userInfo.setArea(row.getString(6)); AdamUser user = new AdamUser();
userInfo.setSignature(row.getString(7)); user.setUid(uid);
userInfo.setAvatar(row.getString(8)); user.setMobile(row.getString(2));
userInfo.setBackground(row.getString(9)); user.setCreatedAt(row.getLocalDateTime(10));
userInfo.setQrCode("lN".concat(userInfo.getUid()).concat(RandomStringUtils.randomAlphanumeric(5).toUpperCase())); user.setUpdatedAt(row.getLocalDateTime(11));
userInfo.setRongCloudToken(row.getString(13)); user.setState(1);
userInfo.setRongCloudTag(row.getInt(14));
AdamUserInfo userInfo = new AdamUserInfo();
AdamUserInfoVo vo = AdamUserInfoVo.getNew(); userInfo.setUid(user.getUid());
vo.setUid(user.getUid()); userInfo.setNickname(row.getString(3));
vo.setMobile(user.getMobile()); userInfo.setSex(JsonUtils.toJson(sex));
vo.setNickname(userInfo.getNickname()); userInfo.setBirthday(row.getLocalDate(5));
vo.setState(user.getState()); userInfo.setArea(row.getString(6));
vo.setSex(sex); userInfo.setSignature(row.getString(7));
vo.setBirthday(DateUtil.Formatter.yyyy_MM_dd.format(userInfo.getBirthday())); userInfo.setAvatar(row.getString(8));
vo.setArea(userInfo.getArea()); userInfo.setBackground(row.getString(9));
vo.setSignature(userInfo.getSignature()); userInfo.setQrCode("lN".concat(userInfo.getUid()).concat(RandomStringUtils.randomAlphanumeric(5).toUpperCase()));
vo.setAvatar(userInfo.getAvatar()); userInfo.setRongCloudToken(row.getString(13));
vo.setBackground(userInfo.getBackground()); userInfo.setRongCloudTag(row.getInt(14));
vo.setCreateAt(user.getCreatedAt());
vo.setUpdatedAt(user.getUpdatedAt()); AdamUserInfoVo vo = AdamUserInfoVo.getNew();
vo.setIsComplete(row.getInt(12)); vo.setUid(user.getUid());
vo.setQrCode(userInfo.getQrCode()); vo.setMobile(user.getMobile());
vo.setNickname(userInfo.getNickname());
userList.add(user); vo.setState(user.getState());
userInfoList.add(userInfo); vo.setSex(sex);
vos.add(vo); vo.setBirthday(DateUtil.Formatter.yyyy_MM_dd.format(userInfo.getBirthday()));
vo.setArea(userInfo.getArea());
vo.setSignature(userInfo.getSignature());
vo.setAvatar(userInfo.getAvatar());
vo.setBackground(userInfo.getBackground());
vo.setCreateAt(user.getCreatedAt());
vo.setUpdatedAt(user.getUpdatedAt());
vo.setIsComplete(row.getInt(12));
vo.setQrCode(userInfo.getQrCode());
userList.add(user);
userInfoList.add(userInfo);
vos.add(vo);
}
if (userList.size() == 500) {
tl += userList.size();
mongoTemplate.insert(vos, AdamUserInfoVo.class.getSimpleName());
log.info("DM.execute.result:{} - {}", adamUserService.saveBatch(userList), adamUserInfoService.saveBatch(userInfoList));
vos.clear();
userList.clear();
userInfoList.clear();
}
} }
} row.close();
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, userList.size()); statement.close();
if (!CollectionUtils.isEmpty(userList)) {
tl += userList.size(); num++;
mongoTemplate.insert(vos, AdamUserInfoVo.class.getSimpleName()); ct -= pSize;
log.info("DM.execute.limit.result:{}", adamUserService.saveBatch(userList));
log.info("DM.execute.limit.result:{}", adamUserInfoService.saveBatch(userInfoList));
}
num++; log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, tl);
ct -= pSize; }
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
} catch (Exception e) {
log.error("ex:getConnection,dG/dg:{}/{},msg:{}", dG, dg, e.getMessage());
}
try {
if (null != row) row.close();
} catch (SQLException ignored) {
}
try {
if (null != statement) statement.close();
} catch (SQLException ignored) {
}
try {
if (null != connection) connection.close();
} catch (SQLException ignored) {
} }
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
row.close();
statement.close();
connection.close();
} }
} }
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMUserMemberProcessor.java
View file @ a1485b47
...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired; ...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
...@@ -48,63 +48,85 @@ public class DMUserMemberProcessor extends DataMigrationProcessorService { ...@@ -48,63 +48,85 @@ public class DMUserMemberProcessor extends DataMigrationProcessorService {
} }
sql += " order by id limit ?,?"; sql += " order by id limit ?,?";
Class.forName(DB_DRIVER); Connection connection = null;
Connection connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD); PreparedStatement statement = null;
ResultSetImpl row = null;
PreparedStatement statement = connection.prepareStatement(sqlCount); try {
ResultSetImpl row = (ResultSetImpl) statement.executeQuery(); Class.forName(DB_DRIVER);
row.first(); connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD);
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct; statement = connection.prepareStatement(sqlCount);
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
while (ct > 0) {
statement = connection.prepareStatement(sql);
int ls = pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} - begin", ls, pSize); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
row.close();
statement.close();
while (ct > 0) {
statement = connection.prepareStatement(sql);
int ls = pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} - begin", ls, pSize);
List<AdamUserMember> userMemberList = new ArrayList<>(); List<AdamUserMember> userMemberList = new ArrayList<>();
List<AdamUserMemberVo> vos = new ArrayList<>(); List<AdamUserMemberVo> vos = new ArrayList<>();
while (row.next()) { while (row.next()) {
String uid = row.getString(1); String uid = row.getString(1);
if (!reUidList.contains(uid)) { if (!reUidList.contains(uid)) {
AdamUserMember userMember = new AdamUserMember(); AdamUserMember userMember = new AdamUserMember();
userMember.setUid(uid); userMember.setUid(uid);
userMember.setMemberId(row.getString(2)); userMember.setMemberId(row.getString(2));
userMember.setMemberNo(row.getString(3)); userMember.setMemberNo(row.getString(3));
userMember.setState(row.getInt(4)); userMember.setState(row.getInt(4));
userMember.setExpiryAt(row.getLocalDateTime(5)); userMember.setExpiryAt(row.getLocalDateTime(5));
userMember.setCreatedAt(row.getLocalDateTime(6)); userMember.setCreatedAt(row.getLocalDateTime(6));
userMember.setUpdatedAt(row.getLocalDateTime(7)); userMember.setUpdatedAt(row.getLocalDateTime(7));
AdamUserMemberVo vo = AdamUserMemberVo.getNew(); AdamUserMemberVo vo = AdamUserMemberVo.getNew();
vo.setUid(uid); vo.setUid(uid);
vo.setMemberId(userMember.getMemberId()); vo.setMemberId(userMember.getMemberId());
vo.setMemberNo(userMember.getMemberNo()); vo.setMemberNo(userMember.getMemberNo());
vo.setState(userMember.getState()); vo.setState(userMember.getState());
vo.setExpiryAt(userMember.getExpiryAt()); vo.setExpiryAt(userMember.getExpiryAt());
vo.setCreatedAt(userMember.getCreatedAt()); vo.setCreatedAt(userMember.getCreatedAt());
vo.setUpdatedAt(userMember.getUpdatedAt()); vo.setUpdatedAt(userMember.getUpdatedAt());
userMemberList.add(userMember); userMemberList.add(userMember);
vos.add(vo); vos.add(vo);
}
if (userMemberList.size() == 500) {
tl += userMemberList.size();
mongoTemplate.insert(vos, AdamUserMemberVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamUserMemberService.saveBatch(userMemberList));
vos.clear();
userMemberList.clear();
}
} }
} row.close();
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, userMemberList.size()); statement.close();
if (!CollectionUtils.isEmpty(userMemberList)) {
tl += userMemberList.size();
mongoTemplate.insert(vos, AdamUserMemberVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamUserMemberService.saveBatch(userMemberList));
}
num++; num++;
ct -= pSize; ct -= pSize;
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, tl);
}
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
} catch (Exception e) {
log.error("ex:getConnection,dG/dg:{}/{},msg:{}", dG, dg, e.getMessage());
}
try {
if (null != row) row.close();
} catch (SQLException ignored) {
}
try {
if (null != statement) statement.close();
} catch (SQLException ignored) {
}
try {
if (null != connection) connection.close();
} catch (SQLException ignored) {
} }
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
row.close();
statement.close();
connection.close();
} }
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment