记得上下班打卡 | git大法好，push需谨慎

Skip to content

L
liquidnet-bus-v1
Commit a1485b47 authored by 张国柄's avatar 张国柄
Browse files
Options

opt dm;

parent c52c780b
Show whitespace changes
Inline Side-by-side
Showing with 829 additions and 632 deletions
liquidnet-bus-config/liquidnet-config/liquidnet-service-adam.yml
View file @ a1485b47
...@@ -118,21 +118,14 @@ spring: ...@@ -118,21 +118,14 @@ spring:
url: jdbc:mysql://${liquidnet.mysql.urlHostAndPort}/${liquidnet.mysql.database-name}?serverTimezone=Asia/Shanghai&characterEncoding=utf-8&useSSL=false url: jdbc:mysql://${liquidnet.mysql.urlHostAndPort}/${liquidnet.mysql.database-name}?serverTimezone=Asia/Shanghai&characterEncoding=utf-8&useSSL=false
username: ${liquidnet.mysql.username} username: ${liquidnet.mysql.username}
password: ${liquidnet.mysql.password} password: ${liquidnet.mysql.password}
# type: org.apache.tomcat.jdbc.pool.DataSource # type: org.apache.tomcat.jdbc.pool.DataSource
driver-class-name: com.mysql.cj.jdbc.Driver driver-class-name: com.mysql.cj.jdbc.Driver
filters: stat hikari:
maxActive: 200 maximum-pool-size: 500
initialSize: 1 # 迁移数据暂设10分钟
maxWait: 60000 connection-timeout: 600000
minIdle: 1 minimum-idle: 10
timeBetweenEvictionRunsMillis: 60000 connection-test-query: SELECT 1
minEvictableIdleTimeMillis: 300000
validationQuery: select 'x'
testWhileIdle: true
testOnBorrow: false
testOnReturn: false
poolPreparedStatements: true
maxOpenPreparedStatements: 20
# ----------------------------------------------------------- # -----------------------------------------------------------
# ----------------------------------------------------------- # -----------------------------------------------------------
......
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/docu/db_ln_adam_initialdata.sql
View file @ a1485b47
...@@ -42,7 +42,7 @@ create table adam_user ...@@ -42,7 +42,7 @@ create table adam_user
( (
mid bigint unsigned auto_increment primary key, mid bigint unsigned auto_increment primary key,
uid varchar(64) not null, uid varchar(64) not null,
mobile varchar(30) not null, mobile varchar(30),
passwd varchar(64), passwd varchar(64),
pay_code varchar(64), pay_code varchar(64),
state tinyint default 0 comment '1-NORMAL,2-INVALID', state tinyint default 0 comment '1-NORMAL,2-INVALID',
......
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/controller/admin/AdamDMAdminController.java
View file @ a1485b47
...@@ -67,6 +67,7 @@ public class AdamDMAdminController extends AdamBaseController { ...@@ -67,6 +67,7 @@ public class AdamDMAdminController extends AdamBaseController {
log.info("==================" + k); log.info("==================" + k);
dmUserInformationProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : ""); dmUserInformationProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : "");
}); });
try { Thread.sleep(100L); } catch (InterruptedException ignored) {}
} }
exec.shutdown(); exec.shutdown();
} else { } else {
...@@ -95,6 +96,7 @@ public class AdamDMAdminController extends AdamBaseController { ...@@ -95,6 +96,7 @@ public class AdamDMAdminController extends AdamBaseController {
log.info("==================" + k); log.info("==================" + k);
dmThirdPartsProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : ""); dmThirdPartsProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : "");
}); });
try { Thread.sleep(100L); } catch (InterruptedException ignored) {}
} }
exec.shutdown(); exec.shutdown();
} else { } else {
...@@ -123,6 +125,7 @@ public class AdamDMAdminController extends AdamBaseController { ...@@ -123,6 +125,7 @@ public class AdamDMAdminController extends AdamBaseController {
log.info("==================" + k); log.info("==================" + k);
dmRealNameProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : ""); dmRealNameProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : "");
}); });
try { Thread.sleep(100L); } catch (InterruptedException ignored) {}
} }
exec.shutdown(); exec.shutdown();
} else { } else {
...@@ -151,6 +154,7 @@ public class AdamDMAdminController extends AdamBaseController { ...@@ -151,6 +154,7 @@ public class AdamDMAdminController extends AdamBaseController {
log.info("==================" + k); log.info("==================" + k);
dmCollectionProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : ""); dmCollectionProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : "");
}); });
try { Thread.sleep(100L); } catch (InterruptedException ignored) {}
} }
exec.shutdown(); exec.shutdown();
} else { } else {
...@@ -179,6 +183,7 @@ public class AdamDMAdminController extends AdamBaseController { ...@@ -179,6 +183,7 @@ public class AdamDMAdminController extends AdamBaseController {
log.info("==================" + k); log.info("==================" + k);
dmEntersProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : ""); dmEntersProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : "");
}); });
try { Thread.sleep(100L); } catch (InterruptedException ignored) {}
} }
exec.shutdown(); exec.shutdown();
} else { } else {
...@@ -207,6 +212,7 @@ public class AdamDMAdminController extends AdamBaseController { ...@@ -207,6 +212,7 @@ public class AdamDMAdminController extends AdamBaseController {
log.info("==================" + k); log.info("==================" + k);
dmAddressesProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : ""); dmAddressesProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : "");
}); });
try { Thread.sleep(100L); } catch (InterruptedException ignored) {}
} }
exec.shutdown(); exec.shutdown();
} else { } else {
...@@ -235,6 +241,7 @@ public class AdamDMAdminController extends AdamBaseController { ...@@ -235,6 +241,7 @@ public class AdamDMAdminController extends AdamBaseController {
log.info("==================" + k); log.info("==================" + k);
dmUserMemberProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : ""); dmUserMemberProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : "");
}); });
try { Thread.sleep(100L); } catch (InterruptedException ignored) {}
} }
exec.shutdown(); exec.shutdown();
} else { } else {
......
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMAddressesProcessor.java
View file @ a1485b47
...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired; ...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
...@@ -47,15 +47,20 @@ public class DMAddressesProcessor extends DataMigrationProcessorService { ...@@ -47,15 +47,20 @@ public class DMAddressesProcessor extends DataMigrationProcessorService {
} }
sql += " order by id limit ?,?"; sql += " order by id limit ?,?";
Connection connection = null;
PreparedStatement statement = null;
ResultSetImpl row = null;
try {
Class.forName(DB_DRIVER); Class.forName(DB_DRIVER);
Connection connection = DriverManager.getConnection(DB_URL + DB_NAME_MALL, DB_USER, DB_PWD); connection = DriverManager.getConnection(DB_URL + DB_NAME_MALL, DB_USER, DB_PWD);
statement = connection.prepareStatement(sqlCount);
PreparedStatement statement = connection.prepareStatement(sqlCount); row = (ResultSetImpl) statement.executeQuery();
ResultSetImpl row = (ResultSetImpl) statement.executeQuery();
row.first(); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct; int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql); log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
row.close();
statement.close();
while (ct > 0) { while (ct > 0) {
statement = connection.prepareStatement(sql); statement = connection.prepareStatement(sql);
int ls = pSize * num; int ls = pSize * num;
...@@ -68,7 +73,6 @@ public class DMAddressesProcessor extends DataMigrationProcessorService { ...@@ -68,7 +73,6 @@ public class DMAddressesProcessor extends DataMigrationProcessorService {
List<AdamAddressesVo> vos = new ArrayList<>(); List<AdamAddressesVo> vos = new ArrayList<>();
while (row.next()) { while (row.next()) {
String uid = row.getString(2); String uid = row.getString(2);
if (!reUidList.contains(uid)) { if (!reUidList.contains(uid)) {
AdamAddresses addresses = new AdamAddresses(); AdamAddresses addresses = new AdamAddresses();
addresses.setAddressesId(row.getString(1)); addresses.setAddressesId(row.getString(1));
...@@ -88,20 +92,37 @@ public class DMAddressesProcessor extends DataMigrationProcessorService { ...@@ -88,20 +92,37 @@ public class DMAddressesProcessor extends DataMigrationProcessorService {
addressesList.add(addresses); addressesList.add(addresses);
vos.add(vo); vos.add(vo);
} }
} if (addressesList.size() == 500) {
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, addressesList.size());
if (!CollectionUtils.isEmpty(addressesList)) {
tl += addressesList.size(); tl += addressesList.size();
mongoTemplate.insert(vos, AdamAddressesVo.class.getSimpleName()); mongoTemplate.insert(vos, AdamAddressesVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamAddressesService.saveBatch(addressesList)); log.info("DM.execute.limit.result:{}", adamAddressesService.saveBatch(addressesList));
vos.clear();
addressesList.clear();
} }
}
row.close();
statement.close();
num++; num++;
ct -= pSize; ct -= pSize;
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, tl);
} }
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl); log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
row.close(); } catch (Exception e) {
statement.close(); log.error("ex:getConnection,dG/dg:{}/{},msg:{}", dG, dg, e.getMessage());
connection.close(); }
try {
if (null != row) row.close();
} catch (SQLException ignored) {
}
try {
if (null != statement) statement.close();
} catch (SQLException ignored) {
}
try {
if (null != connection) connection.close();
} catch (SQLException ignored) {
}
} }
} }
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMCollectionProcessor.java
View file @ a1485b47
...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired; ...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
...@@ -35,7 +35,7 @@ public class DMCollectionProcessor extends DataMigrationProcessorService { ...@@ -35,7 +35,7 @@ public class DMCollectionProcessor extends DataMigrationProcessorService {
log.info("DM.flush.AdamCollectVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").exists(true)), AdamCollectVo.class.getSimpleName()).getDeletedCount()); log.info("DM.flush.AdamCollectVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").exists(true)), AdamCollectVo.class.getSimpleName()).getDeletedCount());
} }
String sqlCount = "select count(1) from user_collections where type='TICKET' and status=1 and content_id>=5722 and created_at"+(StringUtils.isBlank(incrDt) ? "<" : ">=")+"curdate()"; String sqlCount = "select count(1) from user_collections where type='TICKET' and status=1 and content_id>=5722 and created_at" + (StringUtils.isBlank(incrDt) ? "<" : ">=") + "curdate()";
if (null != dg) { if (null != dg) {
sqlCount = sqlCount + " and id%" + dG + "=" + dg; sqlCount = sqlCount + " and id%" + dG + "=" + dg;
} }
...@@ -48,15 +48,20 @@ public class DMCollectionProcessor extends DataMigrationProcessorService { ...@@ -48,15 +48,20 @@ public class DMCollectionProcessor extends DataMigrationProcessorService {
} }
sql += " order by id limit ?,?"; sql += " order by id limit ?,?";
Connection connection = null;
PreparedStatement statement = null;
ResultSetImpl row = null;
try {
Class.forName(DB_DRIVER); Class.forName(DB_DRIVER);
Connection connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD); connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD);
statement = connection.prepareStatement(sqlCount);
PreparedStatement statement = connection.prepareStatement(sqlCount); row = (ResultSetImpl) statement.executeQuery();
ResultSetImpl row = (ResultSetImpl) statement.executeQuery();
row.first(); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct; int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql); log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
row.close();
statement.close();
while (ct > 0) { while (ct > 0) {
statement = connection.prepareStatement(sql); statement = connection.prepareStatement(sql);
int ls = pSize * num; int ls = pSize * num;
...@@ -81,20 +86,37 @@ public class DMCollectionProcessor extends DataMigrationProcessorService { ...@@ -81,20 +86,37 @@ public class DMCollectionProcessor extends DataMigrationProcessorService {
collectionList.add(collection); collectionList.add(collection);
vos.add(AdamCollectVo.getNew().copy(collection)); vos.add(AdamCollectVo.getNew().copy(collection));
} }
} if (collectionList.size() == 500) {
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, collectionList.size());
if (!CollectionUtils.isEmpty(collectionList)) {
tl += collectionList.size(); tl += collectionList.size();
mongoTemplate.insert(vos, AdamCollectVo.class.getSimpleName()); mongoTemplate.insert(vos, AdamCollectVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamCollectionService.saveBatch(collectionList)); log.info("DM.execute.limit.result:{}", adamCollectionService.saveBatch(collectionList));
vos.clear();
collectionList.clear();
}
} }
row.close();
statement.close();
num++; num++;
ct -= pSize; ct -= pSize;
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, tl);
} }
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl); log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
row.close(); } catch (Exception e) {
statement.close(); log.error("ex:getConnection,dG/dg:{}/{},msg:{}", dG, dg, e.getMessage());
connection.close(); }
try {
if (null != row) row.close();
} catch (SQLException ignored) {
}
try {
if (null != statement) statement.close();
} catch (SQLException ignored) {
}
try {
if (null != connection) connection.close();
} catch (SQLException ignored) {
}
} }
} }
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMEntersProcessor.java
View file @ a1485b47
...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired; ...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
...@@ -48,15 +48,20 @@ public class DMEntersProcessor extends DataMigrationProcessorService { ...@@ -48,15 +48,20 @@ public class DMEntersProcessor extends DataMigrationProcessorService {
} }
sql += " order by id limit ?,?"; sql += " order by id limit ?,?";
Connection connection = null;
PreparedStatement statement = null;
ResultSetImpl row = null;
try {
Class.forName(DB_DRIVER); Class.forName(DB_DRIVER);
Connection connection = DriverManager.getConnection(DB_URL + DB_NAME_MALL, DB_USER, DB_PWD); connection = DriverManager.getConnection(DB_URL + DB_NAME_MALL, DB_USER, DB_PWD);
statement = connection.prepareStatement(sqlCount);
PreparedStatement statement = connection.prepareStatement(sqlCount); row = (ResultSetImpl) statement.executeQuery();
ResultSetImpl row = (ResultSetImpl) statement.executeQuery();
row.first(); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct; int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql); log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
row.close();
statement.close();
while (ct > 0) { while (ct > 0) {
statement = connection.prepareStatement(sql); statement = connection.prepareStatement(sql);
int ls = pSize * num; int ls = pSize * num;
...@@ -69,7 +74,6 @@ public class DMEntersProcessor extends DataMigrationProcessorService { ...@@ -69,7 +74,6 @@ public class DMEntersProcessor extends DataMigrationProcessorService {
List<AdamEntersVo> vos = new ArrayList<>(); List<AdamEntersVo> vos = new ArrayList<>();
while (row.next()) { while (row.next()) {
String uid = row.getString(2); String uid = row.getString(2);
if (!reUidList.contains(uid)) { if (!reUidList.contains(uid)) {
AdamEnters enters = new AdamEnters(); AdamEnters enters = new AdamEnters();
enters.setEntersId(row.getString(1)); enters.setEntersId(row.getString(1));
...@@ -87,20 +91,37 @@ public class DMEntersProcessor extends DataMigrationProcessorService { ...@@ -87,20 +91,37 @@ public class DMEntersProcessor extends DataMigrationProcessorService {
entersList.add(enters); entersList.add(enters);
vos.add(vo); vos.add(vo);
} }
} if (entersList.size() == 500) {
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, entersList.size());
if (!CollectionUtils.isEmpty(entersList)) {
tl += entersList.size(); tl += entersList.size();
mongoTemplate.insert(vos, AdamEntersVo.class.getSimpleName()); mongoTemplate.insert(vos, AdamEntersVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamEntersService.saveBatch(entersList)); log.info("DM.execute.limit.result:{}", adamEntersService.saveBatch(entersList));
vos.clear();
entersList.clear();
} }
}
row.close();
statement.close();
num++; num++;
ct -= pSize; ct -= pSize;
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, tl);
} }
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl); log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
row.close(); } catch (Exception e) {
statement.close(); log.error("ex:getConnection,dG/dg:{}/{},msg:{}", dG, dg, e.getMessage());
connection.close(); }
try {
if (null != row) row.close();
} catch (SQLException ignored) {
}
try {
if (null != statement) statement.close();
} catch (SQLException ignored) {
}
try {
if (null != connection) connection.close();
} catch (SQLException ignored) {
}
} }
} }
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMMemberCodeProcessor.java
View file @ a1485b47
...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired; ...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
...@@ -29,154 +29,155 @@ public class DMMemberCodeProcessor extends DataMigrationProcessorService { ...@@ -29,154 +29,155 @@ public class DMMemberCodeProcessor extends DataMigrationProcessorService {
IAdamMemberCodeService adamMemberCodeService; IAdamMemberCodeService adamMemberCodeService;
private static Map<String, String> mMap = new HashMap<>(); private static Map<String, String> mMap = new HashMap<>();
static { static {
mMap.put("78","2159902978359319814V"); mMap.put("78", "2159902978359319814V");
mMap.put("93","2159903004882502855V"); mMap.put("93", "2159903004882502855V");
mMap.put("128","2159903083063113534V"); mMap.put("128", "2159903083063113534V");
mMap.put("171","2159903164279244032V"); mMap.put("171", "2159903164279244032V");
mMap.put("177","2159903178213564580V"); mMap.put("177", "2159903178213564580V");
mMap.put("201","2159903231304794843V"); mMap.put("201", "2159903231304794843V");
mMap.put("224","2159903286921267622V"); mMap.put("224", "2159903286921267622V");
mMap.put("278","2159903315535368620V"); mMap.put("278", "2159903315535368620V");
mMap.put("308","2159903325648681321V"); mMap.put("308", "2159903325648681321V");
mMap.put("371","2159903353285533024V"); mMap.put("371", "2159903353285533024V");
mMap.put("406","2159903365042471904V"); mMap.put("406", "2159903365042471904V");
mMap.put("448","2159903380066742978V"); mMap.put("448", "2159903380066742978V");
mMap.put("470","2159903389019474764V"); mMap.put("470", "2159903389019474764V");
mMap.put("514","2159903402623760508V"); mMap.put("514", "2159903402623760508V");
mMap.put("564","2159903420100480044V"); mMap.put("564", "2159903420100480044V");
mMap.put("566","2159903420815652272V"); mMap.put("566", "2159903420815652272V");
mMap.put("572","2159903425097220662V"); mMap.put("572", "2159903425097220662V");
mMap.put("584","2159903431725878110V"); mMap.put("584", "2159903431725878110V");
mMap.put("621","2159903445678115851V"); mMap.put("621", "2159903445678115851V");
mMap.put("625","2159903448130804563V"); mMap.put("625", "2159903448130804563V");
mMap.put("629","2159903449512021348V"); mMap.put("629", "2159903449512021348V");
mMap.put("634","2159903451797722527V"); mMap.put("634", "2159903451797722527V");
mMap.put("641","2159903456947872759V"); mMap.put("641", "2159903456947872759V");
mMap.put("649","2159903462764769342V"); mMap.put("649", "2159903462764769342V");
mMap.put("663","2159903470899755168V"); mMap.put("663", "2159903470899755168V");
mMap.put("670","2159903474711853726V"); mMap.put("670", "2159903474711853726V");
mMap.put("671","2159903474789135882V"); mMap.put("671", "2159903474789135882V");
mMap.put("692","2159903489229795967V"); mMap.put("692", "2159903489229795967V");
mMap.put("694","2159903491297244057V"); mMap.put("694", "2159903491297244057V");
mMap.put("724","2159903503090588386V"); mMap.put("724", "2159903503090588386V");
mMap.put("744","2159903517757370778V"); mMap.put("744", "2159903517757370778V");
mMap.put("753","2159903521221866688V"); mMap.put("753", "2159903521221866688V");
mMap.put("767","2159903529347024019V"); mMap.put("767", "2159903529347024019V");
mMap.put("812","2159903555573538568V"); mMap.put("812", "2159903555573538568V");
mMap.put("821","2159903560265023540V"); mMap.put("821", "2159903560265023540V");
mMap.put("831","2159903566740588324V"); mMap.put("831", "2159903566740588324V");
mMap.put("847","2159903574058019392V"); mMap.put("847", "2159903574058019392V");
mMap.put("880","2159903599623697117V"); mMap.put("880", "2159903599623697117V");
mMap.put("884","2159903605349638303V"); mMap.put("884", "2159903605349638303V");
mMap.put("893","2159903612326372870V"); mMap.put("893", "2159903612326372870V");
mMap.put("897","2159903630571607844V"); mMap.put("897", "2159903630571607844V");
mMap.put("899","2159903631906318964V"); mMap.put("899", "2159903631906318964V");
mMap.put("915","2159903663070164126V"); mMap.put("915", "2159903663070164126V");
mMap.put("925","2159903708638226752V"); mMap.put("925", "2159903708638226752V");
mMap.put("926","2159903709187446675V"); mMap.put("926", "2159903709187446675V");
mMap.put("929","2159903725057215138V"); mMap.put("929", "2159903725057215138V");
mMap.put("932","2159903733267427426V"); mMap.put("932", "2159903733267427426V");
mMap.put("934","2159903746991129377V"); mMap.put("934", "2159903746991129377V");
mMap.put("945","2159903899372919752V"); mMap.put("945", "2159903899372919752V");
mMap.put("946","2159903944921229059V"); mMap.put("946", "2159903944921229059V");
mMap.put("947","2159903973383580976V"); mMap.put("947", "2159903973383580976V");
mMap.put("948","2159904743264389337V"); mMap.put("948", "2159904743264389337V");
mMap.put("949","2159904881753208429V"); mMap.put("949", "2159904881753208429V");
mMap.put("950","2159905009806735836V"); mMap.put("950", "2159905009806735836V");
mMap.put("951","2159905284996580808V"); mMap.put("951", "2159905284996580808V");
mMap.put("952","2159905286677634113V"); mMap.put("952", "2159905286677634113V");
mMap.put("953","2159905455234006614V"); mMap.put("953", "2159905455234006614V");
mMap.put("954","2159905499349116774V"); mMap.put("954", "2159905499349116774V");
mMap.put("955","2159905536256825603V"); mMap.put("955", "2159905536256825603V");
mMap.put("956","2159906003655765269V"); mMap.put("956", "2159906003655765269V");
mMap.put("957","2159909717834813970V"); mMap.put("957", "2159909717834813970V");
mMap.put("958","2159910092253472736V"); mMap.put("958", "2159910092253472736V");
mMap.put("959","2159910193014350228V"); mMap.put("959", "2159910193014350228V");
mMap.put("960","2159910527370977102V"); mMap.put("960", "2159910527370977102V");
mMap.put("961","2159910531884733969V"); mMap.put("961", "2159910531884733969V");
mMap.put("962","2159910679489574768V"); mMap.put("962", "2159910679489574768V");
mMap.put("963","2159910910443346816V"); mMap.put("963", "2159910910443346816V");
mMap.put("964","2159912826218304603V"); mMap.put("964", "2159912826218304603V");
mMap.put("965","1599558421V"); mMap.put("965", "1599558421V");
mMap.put("966","1599558422V"); mMap.put("966", "1599558422V");
mMap.put("967","1599558423V"); mMap.put("967", "1599558423V");
mMap.put("968","1599558424V"); mMap.put("968", "1599558424V");
mMap.put("969","1599558425V"); mMap.put("969", "1599558425V");
mMap.put("970","1599558426V"); mMap.put("970", "1599558426V");
mMap.put("971","1599558427V"); mMap.put("971", "1599558427V");
mMap.put("972","1599558428V"); mMap.put("972", "1599558428V");
mMap.put("973","1599558429V"); mMap.put("973", "1599558429V");
mMap.put("974","1599558431V"); mMap.put("974", "1599558431V");
mMap.put("975","2159956316098104568V"); mMap.put("975", "2159956316098104568V");
mMap.put("976","2159956355872033001V"); mMap.put("976", "2159956355872033001V");
mMap.put("977","2159956860062026155V"); mMap.put("977", "2159956860062026155V");
mMap.put("978","2159957584538985726V"); mMap.put("978", "2159957584538985726V");
mMap.put("979","2159957623536849859V"); mMap.put("979", "2159957623536849859V");
mMap.put("980","2159957731800533965V"); mMap.put("980", "2159957731800533965V");
mMap.put("981","2159957840680405904V"); mMap.put("981", "2159957840680405904V");
mMap.put("982","2159957929849237090V"); mMap.put("982", "2159957929849237090V");
mMap.put("983","2159962320514458284V"); mMap.put("983", "2159962320514458284V");
mMap.put("984","1599631361V"); mMap.put("984", "1599631361V");
mMap.put("985","1599631362V"); mMap.put("985", "1599631362V");
mMap.put("986","1599631363V"); mMap.put("986", "1599631363V");
mMap.put("987","1599631364V"); mMap.put("987", "1599631364V");
mMap.put("988","1599631365V"); mMap.put("988", "1599631365V");
mMap.put("989","1599631366V"); mMap.put("989", "1599631366V");
mMap.put("990","1599631367V"); mMap.put("990", "1599631367V");
mMap.put("991","1599631368V"); mMap.put("991", "1599631368V");
mMap.put("992","1599631369V"); mMap.put("992", "1599631369V");
mMap.put("993","1599631370V"); mMap.put("993", "1599631370V");
mMap.put("994","2159963377631095182V"); mMap.put("994", "2159963377631095182V");
mMap.put("995","2159963692908504280V"); mMap.put("995", "2159963692908504280V");
mMap.put("996","2159963810907987285V"); mMap.put("996", "2159963810907987285V");
mMap.put("997","2159965666281774447V"); mMap.put("997", "2159965666281774447V");
mMap.put("1000","2159966779640863040V"); mMap.put("1000", "2159966779640863040V");
mMap.put("1062","2159981484289308634V"); mMap.put("1062", "2159981484289308634V");
mMap.put("1104","2159981487011831223V"); mMap.put("1104", "2159981487011831223V");
mMap.put("1114","2159981487541782867V"); mMap.put("1114", "2159981487541782867V");
mMap.put("1198","2159981493699630131V"); mMap.put("1198", "2159981493699630131V");
mMap.put("1200","2159981493787551500V"); mMap.put("1200", "2159981493787551500V");
mMap.put("1235","2159981500360161310V"); mMap.put("1235", "2159981500360161310V");
mMap.put("1262","2159981509978829277V"); mMap.put("1262", "2159981509978829277V");
mMap.put("1287","2159981524026090875V"); mMap.put("1287", "2159981524026090875V");
mMap.put("1304","2159981536177567220V"); mMap.put("1304", "2159981536177567220V");
mMap.put("1310","2159981545302191346V"); mMap.put("1310", "2159981545302191346V");
mMap.put("1319","2159981554253166294V"); mMap.put("1319", "2159981554253166294V");
mMap.put("1325","2159981560137685485V"); mMap.put("1325", "2159981560137685485V");
mMap.put("1329","2159981563803933569V"); mMap.put("1329", "2159981563803933569V");
mMap.put("1335","2159981573925532296V"); mMap.put("1335", "2159981573925532296V");
mMap.put("1390","2159981688651814636V"); mMap.put("1390", "2159981688651814636V");
mMap.put("1391","2159981690863703353V"); mMap.put("1391", "2159981690863703353V");
mMap.put("1428","2159981809289823292V"); mMap.put("1428", "2159981809289823292V");
mMap.put("1430","2159981818758016923V"); mMap.put("1430", "2159981818758016923V");
mMap.put("1458","2159981953122687002V"); mMap.put("1458", "2159981953122687002V");
mMap.put("1494","2159982351398865567V"); mMap.put("1494", "2159982351398865567V");
mMap.put("1495","2159983504307869358V"); mMap.put("1495", "2159983504307869358V");
mMap.put("1496","2159986673135905440V"); mMap.put("1496", "2159986673135905440V");
mMap.put("1497","2159996274042898483V"); mMap.put("1497", "2159996274042898483V");
mMap.put("1498","2160018496723049508V"); mMap.put("1498", "2160018496723049508V");
mMap.put("1499","2160044520193237286V"); mMap.put("1499", "2160044520193237286V");
mMap.put("1500","2160087799997486838V"); mMap.put("1500", "2160087799997486838V");
mMap.put("2230","2160301409593379808V"); mMap.put("2230", "2160301409593379808V");
mMap.put("2231","2160306912844638435V"); mMap.put("2231", "2160306912844638435V");
mMap.put("2232","2160307407349758361V"); mMap.put("2232", "2160307407349758361V");
mMap.put("2233","2160307601583764655V"); mMap.put("2233", "2160307601583764655V");
mMap.put("2234","2160307779536228677V"); mMap.put("2234", "2160307779536228677V");
mMap.put("2492","159963137111V"); mMap.put("2492", "159963137111V");
mMap.put("2510","2160327308412550248V"); mMap.put("2510", "2160327308412550248V");
mMap.put("2515","2160327423681020972V"); mMap.put("2515", "2160327423681020972V");
mMap.put("3359","2160447748642267709V"); mMap.put("3359", "2160447748642267709V");
mMap.put("3360","2160464108681210878V"); mMap.put("3360", "2160464108681210878V");
mMap.put("6315","2161440540017128902V"); mMap.put("6315", "2161440540017128902V");
mMap.put("6511","2161535805626574069V"); mMap.put("6511", "2161535805626574069V");
mMap.put("6726","2161571734236003828V"); mMap.put("6726", "2161571734236003828V");
mMap.put("7629","2161675246646728746V"); mMap.put("7629", "2161675246646728746V");
mMap.put("9150","2161891364918440203V"); mMap.put("9150", "2161891364918440203V");
mMap.put("9554","2161900346985860326V"); mMap.put("9554", "2161900346985860326V");
mMap.put("9888","2162037264105291340V"); mMap.put("9888", "2162037264105291340V");
mMap.put("9889","2162037313688676374V"); mMap.put("9889", "2162037313688676374V");
mMap.put("11470","2162202689894532474V"); mMap.put("11470", "2162202689894532474V");
} }
@SneakyThrows @SneakyThrows
...@@ -198,15 +199,20 @@ public class DMMemberCodeProcessor extends DataMigrationProcessorService { ...@@ -198,15 +199,20 @@ public class DMMemberCodeProcessor extends DataMigrationProcessorService {
String sql = sqlCount.replace("count(1)", field); String sql = sqlCount.replace("count(1)", field);
sql += " order by id limit ?,?"; sql += " order by id limit ?,?";
Connection connection = null;
PreparedStatement statement = null;
ResultSetImpl row = null;
try {
Class.forName(DB_DRIVER); Class.forName(DB_DRIVER);
Connection connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD); connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD);
statement = connection.prepareStatement(sqlCount);
PreparedStatement statement = connection.prepareStatement(sqlCount); row = (ResultSetImpl) statement.executeQuery();
ResultSetImpl row = (ResultSetImpl) statement.executeQuery();
row.first(); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct; int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql); log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
row.close();
statement.close();
while (ct > 0) { while (ct > 0) {
statement = connection.prepareStatement(sql); statement = connection.prepareStatement(sql);
int ls = pSize * num; int ls = pSize * num;
...@@ -240,20 +246,37 @@ public class DMMemberCodeProcessor extends DataMigrationProcessorService { ...@@ -240,20 +246,37 @@ public class DMMemberCodeProcessor extends DataMigrationProcessorService {
memberCodeList.add(memberCode); memberCodeList.add(memberCode);
vos.add(AdamMemberCodeVo.getNew().copy(memberCode)); vos.add(AdamMemberCodeVo.getNew().copy(memberCode));
// } // }
} if (memberCodeList.size() == 500) {
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, memberCodeList.size());
if (!CollectionUtils.isEmpty(memberCodeList)) {
tl += memberCodeList.size(); tl += memberCodeList.size();
mongoTemplate.insert(vos, AdamMemberCodeVo.class.getSimpleName()); mongoTemplate.insert(vos, AdamMemberCodeVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamMemberCodeService.saveBatch(memberCodeList)); log.info("DM.execute.limit.result:{}", adamMemberCodeService.saveBatch(memberCodeList));
vos.clear();
memberCodeList.clear();
} }
}
row.close();
statement.close();
num++; num++;
ct -= pSize; ct -= pSize;
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, tl);
} }
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl); log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
row.close(); } catch (Exception e) {
statement.close(); log.error("ex:getConnection,dG/dg:{}/{},msg:{}", dG, dg, e.getMessage());
connection.close(); }
try {
if (null != row) row.close();
} catch (SQLException ignored) {
}
try {
if (null != statement) statement.close();
} catch (SQLException ignored) {
}
try {
if (null != connection) connection.close();
} catch (SQLException ignored) {
}
} }
} }
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMMemberOrderProcessor.java
View file @ a1485b47
...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired; ...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
...@@ -73,15 +73,20 @@ public class DMMemberOrderProcessor extends DataMigrationProcessorService { ...@@ -73,15 +73,20 @@ public class DMMemberOrderProcessor extends DataMigrationProcessorService {
} }
sql += " order by id limit ?,?"; sql += " order by id limit ?,?";
Connection connection = null;
PreparedStatement statement = null;
ResultSetImpl row = null;
try {
Class.forName(DB_DRIVER); Class.forName(DB_DRIVER);
Connection connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD); connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD);
statement = connection.prepareStatement(sqlCount);
PreparedStatement statement = connection.prepareStatement(sqlCount); row = (ResultSetImpl) statement.executeQuery();
ResultSetImpl row = (ResultSetImpl) statement.executeQuery();
row.first(); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct; int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql); log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
row.close();
statement.close();
while (ct > 0) { while (ct > 0) {
statement = connection.prepareStatement(sql); statement = connection.prepareStatement(sql);
int ls = pSize * num; int ls = pSize * num;
...@@ -124,20 +129,37 @@ public class DMMemberOrderProcessor extends DataMigrationProcessorService { ...@@ -124,20 +129,37 @@ public class DMMemberOrderProcessor extends DataMigrationProcessorService {
memberOrderList.add(memberOrder); memberOrderList.add(memberOrder);
vos.add(vo); vos.add(vo);
} }
} if (memberOrderList.size() == 500) {
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, memberOrderList.size());
if (!CollectionUtils.isEmpty(memberOrderList)) {
tl += memberOrderList.size(); tl += memberOrderList.size();
mongoTemplate.insert(vos, AdamMemberOrderVo.class.getSimpleName()); mongoTemplate.insert(vos, AdamMemberOrderVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamMemberOrderService.saveBatch(memberOrderList)); log.info("DM.execute.limit.result:{}", adamMemberOrderService.saveBatch(memberOrderList));
vos.clear();
memberOrderList.clear();
}
} }
row.close();
statement.close();
num++; num++;
ct -= pSize; ct -= pSize;
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, tl);
} }
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl); log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
row.close(); } catch (Exception e) {
statement.close(); log.error("ex:getConnection,dG/dg:{}/{},msg:{}", dG, dg, e.getMessage());
connection.close(); }
try {
if (null != row) row.close();
} catch (SQLException ignored) {
}
try {
if (null != statement) statement.close();
} catch (SQLException ignored) {
}
try {
if (null != connection) connection.close();
} catch (SQLException ignored) {
}
} }
} }
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMRealNameProcessor.java
View file @ a1485b47
...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired; ...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
...@@ -48,15 +48,20 @@ public class DMRealNameProcessor extends DataMigrationProcessorService { ...@@ -48,15 +48,20 @@ public class DMRealNameProcessor extends DataMigrationProcessorService {
} }
sql += " order by id limit ?,?"; sql += " order by id limit ?,?";
Connection connection = null;
PreparedStatement statement = null;
ResultSetImpl row = null;
try {
Class.forName(DB_DRIVER); Class.forName(DB_DRIVER);
Connection connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD); connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD);
statement = connection.prepareStatement(sqlCount);
PreparedStatement statement = connection.prepareStatement(sqlCount); row = (ResultSetImpl) statement.executeQuery();
ResultSetImpl row = (ResultSetImpl) statement.executeQuery();
row.first(); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct; int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql); log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
row.close();
statement.close();
while (ct > 0) { while (ct > 0) {
statement = connection.prepareStatement(sql); statement = connection.prepareStatement(sql);
int ls = pSize * num; int ls = pSize * num;
...@@ -90,20 +95,37 @@ public class DMRealNameProcessor extends DataMigrationProcessorService { ...@@ -90,20 +95,37 @@ public class DMRealNameProcessor extends DataMigrationProcessorService {
realNameList.add(realName); realNameList.add(realName);
vos.add(vo); vos.add(vo);
} }
} if (realNameList.size() == 500) {
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, realNameList.size());
if (!CollectionUtils.isEmpty(realNameList)) {
tl += realNameList.size(); tl += realNameList.size();
mongoTemplate.insert(vos, AdamRealInfoVo.class.getSimpleName()); mongoTemplate.insert(vos, AdamRealInfoVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamRealNameService.saveBatch(realNameList)); log.info("DM.execute.limit.result:{}", adamRealNameService.saveBatch(realNameList));
vos.clear();
realNameList.clear();
}
} }
row.close();
statement.close();
num++; num++;
ct -= pSize; ct -= pSize;
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, tl);
} }
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl); log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
row.close(); } catch (Exception e) {
statement.close(); log.error("ex:getConnection,dG/dg:{}/{},msg:{}", dG, dg, e.getMessage());
connection.close(); }
try {
if (null != row) row.close();
} catch (SQLException ignored) {
}
try {
if (null != statement) statement.close();
} catch (SQLException ignored) {
}
try {
if (null != connection) connection.close();
} catch (SQLException ignored) {
}
} }
} }
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMThirdPartsProcessor.java
View file @ a1485b47
...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired; ...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
...@@ -36,7 +36,7 @@ public class DMThirdPartsProcessor extends DataMigrationProcessorService { ...@@ -36,7 +36,7 @@ public class DMThirdPartsProcessor extends DataMigrationProcessorService {
} }
String sqlCount = "select count(1) from (\n" + String sqlCount = "select count(1) from (\n" +
" select row_number() over (partition by uid,type order by created_at desc) rn,id,uid,open_id,avatar,nickname,type,created_at,updated_at from user_third_parts where `status`=1 and delete_tag in (0,1) and open_id<>'' and uid>0 and created_at"+(StringUtils.isBlank(incrDt) ? "<" : ">=")+"curdate()\n" + " select row_number() over (partition by uid,type order by created_at desc) rn,id,uid,open_id,avatar,nickname,type,created_at,updated_at from user_third_parts where `status`=1 and delete_tag in (0,1) and open_id<>'' and uid>0 and created_at" + (StringUtils.isBlank(incrDt) ? "<" : ">=") + "curdate()\n" +
" ) t where t.rn=1"; " ) t where t.rn=1";
if (null != dg) { if (null != dg) {
sqlCount = sqlCount + " and id%" + dG + "=" + dg; sqlCount = sqlCount + " and id%" + dG + "=" + dg;
...@@ -50,15 +50,20 @@ public class DMThirdPartsProcessor extends DataMigrationProcessorService { ...@@ -50,15 +50,20 @@ public class DMThirdPartsProcessor extends DataMigrationProcessorService {
} }
sql += " order by id limit ?,?"; sql += " order by id limit ?,?";
Connection connection = null;
PreparedStatement statement = null;
ResultSetImpl row = null;
try {
Class.forName(DB_DRIVER); Class.forName(DB_DRIVER);
Connection connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD); connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD);
statement = connection.prepareStatement(sqlCount);
PreparedStatement statement = connection.prepareStatement(sqlCount); row = (ResultSetImpl) statement.executeQuery();
ResultSetImpl row = (ResultSetImpl) statement.executeQuery();
row.first(); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct; int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql); log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
row.close();
statement.close();
while (ct > 0) { while (ct > 0) {
statement = connection.prepareStatement(sql); statement = connection.prepareStatement(sql);
int ls = pSize * num; int ls = pSize * num;
...@@ -95,20 +100,37 @@ public class DMThirdPartsProcessor extends DataMigrationProcessorService { ...@@ -95,20 +100,37 @@ public class DMThirdPartsProcessor extends DataMigrationProcessorService {
thirdPartyList.add(thirdParty); thirdPartyList.add(thirdParty);
vos.add(vo); vos.add(vo);
} }
} if (thirdPartyList.size() == 500) {
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, thirdPartyList.size());
if (!CollectionUtils.isEmpty(thirdPartyList)) {
tl += thirdPartyList.size(); tl += thirdPartyList.size();
mongoTemplate.insert(vos, AdamThirdPartInfoVo.class.getSimpleName()); mongoTemplate.insert(vos, AdamThirdPartInfoVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamThirdPartyService.saveBatch(thirdPartyList)); log.info("DM.execute.limit.result:{}", adamThirdPartyService.saveBatch(thirdPartyList));
vos.clear();
thirdPartyList.clear();
}
} }
row.close();
statement.close();
num++; num++;
ct -= pSize; ct -= pSize;
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, tl);
} }
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl); log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
row.close(); } catch (Exception e) {
statement.close(); log.error("ex:getConnection,dG/dg:{}/{},msg:{}", dG, dg, e.getMessage());
connection.close(); }
try {
if (null != row) row.close();
} catch (SQLException ignored) {
}
try {
if (null != statement) statement.close();
} catch (SQLException ignored) {
}
try {
if (null != connection) connection.close();
} catch (SQLException ignored) {
}
} }
} }
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMUserInformationProcessor.java
View file @ a1485b47
...@@ -18,11 +18,11 @@ import org.springframework.beans.factory.annotation.Autowired; ...@@ -18,11 +18,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
...@@ -68,15 +68,20 @@ public class DMUserInformationProcessor extends DataMigrationProcessorService { ...@@ -68,15 +68,20 @@ public class DMUserInformationProcessor extends DataMigrationProcessorService {
} }
sql += " order by uid limit ?,?"; sql += " order by uid limit ?,?";
Connection connection = null;
PreparedStatement statement = null;
ResultSetImpl row = null;
try {
Class.forName(DB_DRIVER); Class.forName(DB_DRIVER);
Connection connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD); connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD);
statement = connection.prepareStatement(sqlCount);
PreparedStatement statement = connection.prepareStatement(sqlCount); row = (ResultSetImpl) statement.executeQuery();
ResultSetImpl row = (ResultSetImpl) statement.executeQuery();
row.first(); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct; int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql); log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
row.close();
statement.close();
while (ct > 0) { while (ct > 0) {
statement = connection.prepareStatement(sql); statement = connection.prepareStatement(sql);
int ls = pSize * num; int ls = pSize * num;
...@@ -132,21 +137,38 @@ public class DMUserInformationProcessor extends DataMigrationProcessorService { ...@@ -132,21 +137,38 @@ public class DMUserInformationProcessor extends DataMigrationProcessorService {
userInfoList.add(userInfo); userInfoList.add(userInfo);
vos.add(vo); vos.add(vo);
} }
} if (userList.size() == 500) {
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, userList.size());
if (!CollectionUtils.isEmpty(userList)) {
tl += userList.size(); tl += userList.size();
mongoTemplate.insert(vos, AdamUserInfoVo.class.getSimpleName()); mongoTemplate.insert(vos, AdamUserInfoVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamUserService.saveBatch(userList)); log.info("DM.execute.result:{} - {}", adamUserService.saveBatch(userList), adamUserInfoService.saveBatch(userInfoList));
log.info("DM.execute.limit.result:{}", adamUserInfoService.saveBatch(userInfoList)); vos.clear();
userList.clear();
userInfoList.clear();
}
} }
row.close();
statement.close();
num++; num++;
ct -= pSize; ct -= pSize;
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, tl);
} }
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl); log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
row.close(); } catch (Exception e) {
statement.close(); log.error("ex:getConnection,dG/dg:{}/{},msg:{}", dG, dg, e.getMessage());
connection.close(); }
try {
if (null != row) row.close();
} catch (SQLException ignored) {
}
try {
if (null != statement) statement.close();
} catch (SQLException ignored) {
}
try {
if (null != connection) connection.close();
} catch (SQLException ignored) {
}
} }
} }
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMUserMemberProcessor.java
View file @ a1485b47
...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired; ...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
...@@ -48,15 +48,20 @@ public class DMUserMemberProcessor extends DataMigrationProcessorService { ...@@ -48,15 +48,20 @@ public class DMUserMemberProcessor extends DataMigrationProcessorService {
} }
sql += " order by id limit ?,?"; sql += " order by id limit ?,?";
Connection connection = null;
PreparedStatement statement = null;
ResultSetImpl row = null;
try {
Class.forName(DB_DRIVER); Class.forName(DB_DRIVER);
Connection connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD); connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD);
statement = connection.prepareStatement(sqlCount);
PreparedStatement statement = connection.prepareStatement(sqlCount); row = (ResultSetImpl) statement.executeQuery();
ResultSetImpl row = (ResultSetImpl) statement.executeQuery();
row.first(); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct; int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql); log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
row.close();
statement.close();
while (ct > 0) { while (ct > 0) {
statement = connection.prepareStatement(sql); statement = connection.prepareStatement(sql);
int ls = pSize * num; int ls = pSize * num;
...@@ -91,20 +96,37 @@ public class DMUserMemberProcessor extends DataMigrationProcessorService { ...@@ -91,20 +96,37 @@ public class DMUserMemberProcessor extends DataMigrationProcessorService {
userMemberList.add(userMember); userMemberList.add(userMember);
vos.add(vo); vos.add(vo);
} }
} if (userMemberList.size() == 500) {
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, userMemberList.size());
if (!CollectionUtils.isEmpty(userMemberList)) {
tl += userMemberList.size(); tl += userMemberList.size();
mongoTemplate.insert(vos, AdamUserMemberVo.class.getSimpleName()); mongoTemplate.insert(vos, AdamUserMemberVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamUserMemberService.saveBatch(userMemberList)); log.info("DM.execute.limit.result:{}", adamUserMemberService.saveBatch(userMemberList));
vos.clear();
userMemberList.clear();
}
} }
row.close();
statement.close();
num++; num++;
ct -= pSize; ct -= pSize;
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, tl);
} }
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl); log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
row.close(); } catch (Exception e) {
statement.close(); log.error("ex:getConnection,dG/dg:{}/{},msg:{}", dG, dg, e.getMessage());
connection.close(); }
try {
if (null != row) row.close();
} catch (SQLException ignored) {
}
try {
if (null != statement) statement.close();
} catch (SQLException ignored) {
}
try {
if (null != connection) connection.close();
} catch (SQLException ignored) {
}
} }
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment