记得上下班打卡 | git大法好，push需谨慎

Skip to content

L
liquidnet-bus-v1
Commit a1485b47 authored by 张国柄's avatar 张国柄
Browse files
Options

opt dm;

parent c52c780b
Expand all Show whitespace changes
Inline Side-by-side
Showing with 829 additions and 632 deletions
liquidnet-bus-config/liquidnet-config/liquidnet-service-adam.yml
View file @ a1485b47
......@@ -118,21 +118,14 @@ spring:
url: jdbc:mysql://${liquidnet.mysql.urlHostAndPort}/${liquidnet.mysql.database-name}?serverTimezone=Asia/Shanghai&characterEncoding=utf-8&useSSL=false
username: ${liquidnet.mysql.username}
password: ${liquidnet.mysql.password}
# type: org.apache.tomcat.jdbc.pool.DataSource
# type: org.apache.tomcat.jdbc.pool.DataSource
driver-class-name: com.mysql.cj.jdbc.Driver
filters: stat
maxActive: 200
initialSize: 1
maxWait: 60000
minIdle: 1
timeBetweenEvictionRunsMillis: 60000
minEvictableIdleTimeMillis: 300000
validationQuery: select 'x'
testWhileIdle: true
testOnBorrow: false
testOnReturn: false
poolPreparedStatements: true
maxOpenPreparedStatements: 20
hikari:
maximum-pool-size: 500
# 迁移数据暂设10分钟
connection-timeout: 600000
minimum-idle: 10
connection-test-query: SELECT 1
# -----------------------------------------------------------
# -----------------------------------------------------------
......
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/docu/db_ln_adam_initialdata.sql
View file @ a1485b47
......@@ -42,7 +42,7 @@ create table adam_user
(
mid bigint unsigned auto_increment primary key,
uid varchar(64) not null,
mobile varchar(30) not null,
mobile varchar(30),
passwd varchar(64),
pay_code varchar(64),
state tinyint default 0 comment '1-NORMAL,2-INVALID',
......
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/controller/admin/AdamDMAdminController.java
View file @ a1485b47
......@@ -67,6 +67,7 @@ public class AdamDMAdminController extends AdamBaseController {
log.info("==================" + k);
dmUserInformationProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : "");
});
try { Thread.sleep(100L); } catch (InterruptedException ignored) {}
}
exec.shutdown();
} else {
......@@ -95,6 +96,7 @@ public class AdamDMAdminController extends AdamBaseController {
log.info("==================" + k);
dmThirdPartsProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : "");
});
try { Thread.sleep(100L); } catch (InterruptedException ignored) {}
}
exec.shutdown();
} else {
......@@ -123,6 +125,7 @@ public class AdamDMAdminController extends AdamBaseController {
log.info("==================" + k);
dmRealNameProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : "");
});
try { Thread.sleep(100L); } catch (InterruptedException ignored) {}
}
exec.shutdown();
} else {
......@@ -151,6 +154,7 @@ public class AdamDMAdminController extends AdamBaseController {
log.info("==================" + k);
dmCollectionProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : "");
});
try { Thread.sleep(100L); } catch (InterruptedException ignored) {}
}
exec.shutdown();
} else {
......@@ -179,6 +183,7 @@ public class AdamDMAdminController extends AdamBaseController {
log.info("==================" + k);
dmEntersProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : "");
});
try { Thread.sleep(100L); } catch (InterruptedException ignored) {}
}
exec.shutdown();
} else {
......@@ -207,6 +212,7 @@ public class AdamDMAdminController extends AdamBaseController {
log.info("==================" + k);
dmAddressesProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : "");
});
try { Thread.sleep(100L); } catch (InterruptedException ignored) {}
}
exec.shutdown();
} else {
......@@ -235,6 +241,7 @@ public class AdamDMAdminController extends AdamBaseController {
log.info("==================" + k);
dmUserMemberProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : "");
});
try { Thread.sleep(100L); } catch (InterruptedException ignored) {}
}
exec.shutdown();
} else {
......
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMAddressesProcessor.java
View file @ a1485b47
......@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
......@@ -47,15 +47,20 @@ public class DMAddressesProcessor extends DataMigrationProcessorService {
}
sql += " order by id limit ?,?";
Connection connection = null;
PreparedStatement statement = null;
ResultSetImpl row = null;
try {
Class.forName(DB_DRIVER);
Connection connection = DriverManager.getConnection(DB_URL + DB_NAME_MALL, DB_USER, DB_PWD);
PreparedStatement statement = connection.prepareStatement(sqlCount);
ResultSetImpl row = (ResultSetImpl) statement.executeQuery();
connection = DriverManager.getConnection(DB_URL + DB_NAME_MALL, DB_USER, DB_PWD);
statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery();
row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
row.close();
statement.close();
while (ct > 0) {
statement = connection.prepareStatement(sql);
int ls = pSize * num;
......@@ -68,7 +73,6 @@ public class DMAddressesProcessor extends DataMigrationProcessorService {
List<AdamAddressesVo> vos = new ArrayList<>();
while (row.next()) {
String uid = row.getString(2);
if (!reUidList.contains(uid)) {
AdamAddresses addresses = new AdamAddresses();
addresses.setAddressesId(row.getString(1));
......@@ -88,20 +92,37 @@ public class DMAddressesProcessor extends DataMigrationProcessorService {
addressesList.add(addresses);
vos.add(vo);
}
}
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, addressesList.size());
if (!CollectionUtils.isEmpty(addressesList)) {
if (addressesList.size() == 500) {
tl += addressesList.size();
mongoTemplate.insert(vos, AdamAddressesVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamAddressesService.saveBatch(addressesList));
vos.clear();
addressesList.clear();
}
}
row.close();
statement.close();
num++;
ct -= pSize;
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, tl);
}
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
row.close();
statement.close();
connection.close();
} catch (Exception e) {
log.error("ex:getConnection,dG/dg:{}/{},msg:{}", dG, dg, e.getMessage());
}
try {
if (null != row) row.close();
} catch (SQLException ignored) {
}
try {
if (null != statement) statement.close();
} catch (SQLException ignored) {
}
try {
if (null != connection) connection.close();
} catch (SQLException ignored) {
}
}
}
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMCollectionProcessor.java
View file @ a1485b47
......@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
......@@ -35,7 +35,7 @@ public class DMCollectionProcessor extends DataMigrationProcessorService {
log.info("DM.flush.AdamCollectVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").exists(true)), AdamCollectVo.class.getSimpleName()).getDeletedCount());
}
String sqlCount = "select count(1) from user_collections where type='TICKET' and status=1 and content_id>=5722 and created_at"+(StringUtils.isBlank(incrDt) ? "<" : ">=")+"curdate()";
String sqlCount = "select count(1) from user_collections where type='TICKET' and status=1 and content_id>=5722 and created_at" + (StringUtils.isBlank(incrDt) ? "<" : ">=") + "curdate()";
if (null != dg) {
sqlCount = sqlCount + " and id%" + dG + "=" + dg;
}
......@@ -48,15 +48,20 @@ public class DMCollectionProcessor extends DataMigrationProcessorService {
}
sql += " order by id limit ?,?";
Connection connection = null;
PreparedStatement statement = null;
ResultSetImpl row = null;
try {
Class.forName(DB_DRIVER);
Connection connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD);
PreparedStatement statement = connection.prepareStatement(sqlCount);
ResultSetImpl row = (ResultSetImpl) statement.executeQuery();
connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD);
statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery();
row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
row.close();
statement.close();
while (ct > 0) {
statement = connection.prepareStatement(sql);
int ls = pSize * num;
......@@ -81,20 +86,37 @@ public class DMCollectionProcessor extends DataMigrationProcessorService {
collectionList.add(collection);
vos.add(AdamCollectVo.getNew().copy(collection));
}
}
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, collectionList.size());
if (!CollectionUtils.isEmpty(collectionList)) {
if (collectionList.size() == 500) {
tl += collectionList.size();
mongoTemplate.insert(vos, AdamCollectVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamCollectionService.saveBatch(collectionList));
vos.clear();
collectionList.clear();
}
}
row.close();
statement.close();
num++;
ct -= pSize;
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, tl);
}
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
row.close();
statement.close();
connection.close();
} catch (Exception e) {
log.error("ex:getConnection,dG/dg:{}/{},msg:{}", dG, dg, e.getMessage());
}
try {
if (null != row) row.close();
} catch (SQLException ignored) {
}
try {
if (null != statement) statement.close();
} catch (SQLException ignored) {
}
try {
if (null != connection) connection.close();
} catch (SQLException ignored) {
}
}
}
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMEntersProcessor.java
View file @ a1485b47
......@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
......@@ -48,15 +48,20 @@ public class DMEntersProcessor extends DataMigrationProcessorService {
}
sql += " order by id limit ?,?";
Connection connection = null;
PreparedStatement statement = null;
ResultSetImpl row = null;
try {
Class.forName(DB_DRIVER);
Connection connection = DriverManager.getConnection(DB_URL + DB_NAME_MALL, DB_USER, DB_PWD);
PreparedStatement statement = connection.prepareStatement(sqlCount);
ResultSetImpl row = (ResultSetImpl) statement.executeQuery();
connection = DriverManager.getConnection(DB_URL + DB_NAME_MALL, DB_USER, DB_PWD);
statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery();
row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
row.close();
statement.close();
while (ct > 0) {
statement = connection.prepareStatement(sql);
int ls = pSize * num;
......@@ -69,7 +74,6 @@ public class DMEntersProcessor extends DataMigrationProcessorService {
List<AdamEntersVo> vos = new ArrayList<>();
while (row.next()) {
String uid = row.getString(2);
if (!reUidList.contains(uid)) {
AdamEnters enters = new AdamEnters();
enters.setEntersId(row.getString(1));
......@@ -87,20 +91,37 @@ public class DMEntersProcessor extends DataMigrationProcessorService {
entersList.add(enters);
vos.add(vo);
}
}
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, entersList.size());
if (!CollectionUtils.isEmpty(entersList)) {
if (entersList.size() == 500) {
tl += entersList.size();
mongoTemplate.insert(vos, AdamEntersVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamEntersService.saveBatch(entersList));
vos.clear();
entersList.clear();
}
}
row.close();
statement.close();
num++;
ct -= pSize;
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, tl);
}
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
row.close();
statement.close();
connection.close();
} catch (Exception e) {
log.error("ex:getConnection,dG/dg:{}/{},msg:{}", dG, dg, e.getMessage());
}
try {
if (null != row) row.close();
} catch (SQLException ignored) {
}
try {
if (null != statement) statement.close();
} catch (SQLException ignored) {
}
try {
if (null != connection) connection.close();
} catch (SQLException ignored) {
}
}
}
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMMemberOrderProcessor.java
View file @ a1485b47
......@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
......@@ -73,15 +73,20 @@ public class DMMemberOrderProcessor extends DataMigrationProcessorService {
}
sql += " order by id limit ?,?";
Connection connection = null;
PreparedStatement statement = null;
ResultSetImpl row = null;
try {
Class.forName(DB_DRIVER);
Connection connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD);
PreparedStatement statement = connection.prepareStatement(sqlCount);
ResultSetImpl row = (ResultSetImpl) statement.executeQuery();
connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD);
statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery();
row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
row.close();
statement.close();
while (ct > 0) {
statement = connection.prepareStatement(sql);
int ls = pSize * num;
......@@ -124,20 +129,37 @@ public class DMMemberOrderProcessor extends DataMigrationProcessorService {
memberOrderList.add(memberOrder);
vos.add(vo);
}
}
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, memberOrderList.size());
if (!CollectionUtils.isEmpty(memberOrderList)) {
if (memberOrderList.size() == 500) {
tl += memberOrderList.size();
mongoTemplate.insert(vos, AdamMemberOrderVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamMemberOrderService.saveBatch(memberOrderList));
vos.clear();
memberOrderList.clear();
}
}
row.close();
statement.close();
num++;
ct -= pSize;
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, tl);
}
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
row.close();
statement.close();
connection.close();
} catch (Exception e) {
log.error("ex:getConnection,dG/dg:{}/{},msg:{}", dG, dg, e.getMessage());
}
try {
if (null != row) row.close();
} catch (SQLException ignored) {
}
try {
if (null != statement) statement.close();
} catch (SQLException ignored) {
}
try {
if (null != connection) connection.close();
} catch (SQLException ignored) {
}
}
}
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMRealNameProcessor.java
View file @ a1485b47
......@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
......@@ -48,15 +48,20 @@ public class DMRealNameProcessor extends DataMigrationProcessorService {
}
sql += " order by id limit ?,?";
Connection connection = null;
PreparedStatement statement = null;
ResultSetImpl row = null;
try {
Class.forName(DB_DRIVER);
Connection connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD);
PreparedStatement statement = connection.prepareStatement(sqlCount);
ResultSetImpl row = (ResultSetImpl) statement.executeQuery();
connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD);
statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery();
row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
row.close();
statement.close();
while (ct > 0) {
statement = connection.prepareStatement(sql);
int ls = pSize * num;
......@@ -90,20 +95,37 @@ public class DMRealNameProcessor extends DataMigrationProcessorService {
realNameList.add(realName);
vos.add(vo);
}
}
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, realNameList.size());
if (!CollectionUtils.isEmpty(realNameList)) {
if (realNameList.size() == 500) {
tl += realNameList.size();
mongoTemplate.insert(vos, AdamRealInfoVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamRealNameService.saveBatch(realNameList));
vos.clear();
realNameList.clear();
}
}
row.close();
statement.close();
num++;
ct -= pSize;
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, tl);
}
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
row.close();
statement.close();
connection.close();
} catch (Exception e) {
log.error("ex:getConnection,dG/dg:{}/{},msg:{}", dG, dg, e.getMessage());
}
try {
if (null != row) row.close();
} catch (SQLException ignored) {
}
try {
if (null != statement) statement.close();
} catch (SQLException ignored) {
}
try {
if (null != connection) connection.close();
} catch (SQLException ignored) {
}
}
}
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMThirdPartsProcessor.java
View file @ a1485b47
......@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
......@@ -36,7 +36,7 @@ public class DMThirdPartsProcessor extends DataMigrationProcessorService {
}
String sqlCount = "select count(1) from (\n" +
" select row_number() over (partition by uid,type order by created_at desc) rn,id,uid,open_id,avatar,nickname,type,created_at,updated_at from user_third_parts where `status`=1 and delete_tag in (0,1) and open_id<>'' and uid>0 and created_at"+(StringUtils.isBlank(incrDt) ? "<" : ">=")+"curdate()\n" +
" select row_number() over (partition by uid,type order by created_at desc) rn,id,uid,open_id,avatar,nickname,type,created_at,updated_at from user_third_parts where `status`=1 and delete_tag in (0,1) and open_id<>'' and uid>0 and created_at" + (StringUtils.isBlank(incrDt) ? "<" : ">=") + "curdate()\n" +
" ) t where t.rn=1";
if (null != dg) {
sqlCount = sqlCount + " and id%" + dG + "=" + dg;
......@@ -50,15 +50,20 @@ public class DMThirdPartsProcessor extends DataMigrationProcessorService {
}
sql += " order by id limit ?,?";
Connection connection = null;
PreparedStatement statement = null;
ResultSetImpl row = null;
try {
Class.forName(DB_DRIVER);
Connection connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD);
PreparedStatement statement = connection.prepareStatement(sqlCount);
ResultSetImpl row = (ResultSetImpl) statement.executeQuery();
connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD);
statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery();
row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
row.close();
statement.close();
while (ct > 0) {
statement = connection.prepareStatement(sql);
int ls = pSize * num;
......@@ -95,20 +100,37 @@ public class DMThirdPartsProcessor extends DataMigrationProcessorService {
thirdPartyList.add(thirdParty);
vos.add(vo);
}
}
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, thirdPartyList.size());
if (!CollectionUtils.isEmpty(thirdPartyList)) {
if (thirdPartyList.size() == 500) {
tl += thirdPartyList.size();
mongoTemplate.insert(vos, AdamThirdPartInfoVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamThirdPartyService.saveBatch(thirdPartyList));
vos.clear();
thirdPartyList.clear();
}
}
row.close();
statement.close();
num++;
ct -= pSize;
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, tl);
}
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
row.close();
statement.close();
connection.close();
} catch (Exception e) {
log.error("ex:getConnection,dG/dg:{}/{},msg:{}", dG, dg, e.getMessage());
}
try {
if (null != row) row.close();
} catch (SQLException ignored) {
}
try {
if (null != statement) statement.close();
} catch (SQLException ignored) {
}
try {
if (null != connection) connection.close();
} catch (SQLException ignored) {
}
}
}
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMUserInformationProcessor.java
View file @ a1485b47
......@@ -18,11 +18,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
......@@ -68,15 +68,20 @@ public class DMUserInformationProcessor extends DataMigrationProcessorService {
}
sql += " order by uid limit ?,?";
Connection connection = null;
PreparedStatement statement = null;
ResultSetImpl row = null;
try {
Class.forName(DB_DRIVER);
Connection connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD);
PreparedStatement statement = connection.prepareStatement(sqlCount);
ResultSetImpl row = (ResultSetImpl) statement.executeQuery();
connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD);
statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery();
row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
row.close();
statement.close();
while (ct > 0) {
statement = connection.prepareStatement(sql);
int ls = pSize * num;
......@@ -132,21 +137,38 @@ public class DMUserInformationProcessor extends DataMigrationProcessorService {
userInfoList.add(userInfo);
vos.add(vo);
}
}
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, userList.size());
if (!CollectionUtils.isEmpty(userList)) {
if (userList.size() == 500) {
tl += userList.size();
mongoTemplate.insert(vos, AdamUserInfoVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamUserService.saveBatch(userList));
log.info("DM.execute.limit.result:{}", adamUserInfoService.saveBatch(userInfoList));
log.info("DM.execute.result:{} - {}", adamUserService.saveBatch(userList), adamUserInfoService.saveBatch(userInfoList));
vos.clear();
userList.clear();
userInfoList.clear();
}
}
row.close();
statement.close();
num++;
ct -= pSize;
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, tl);
}
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
row.close();
statement.close();
connection.close();
} catch (Exception e) {
log.error("ex:getConnection,dG/dg:{}/{},msg:{}", dG, dg, e.getMessage());
}
try {
if (null != row) row.close();
} catch (SQLException ignored) {
}
try {
if (null != statement) statement.close();
} catch (SQLException ignored) {
}
try {
if (null != connection) connection.close();
} catch (SQLException ignored) {
}
}
}
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMUserMemberProcessor.java
View file @ a1485b47
......@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
......@@ -48,15 +48,20 @@ public class DMUserMemberProcessor extends DataMigrationProcessorService {
}
sql += " order by id limit ?,?";
Connection connection = null;
PreparedStatement statement = null;
ResultSetImpl row = null;
try {
Class.forName(DB_DRIVER);
Connection connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD);
PreparedStatement statement = connection.prepareStatement(sqlCount);
ResultSetImpl row = (ResultSetImpl) statement.executeQuery();
connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD);
statement = connection.prepareStatement(sqlCount);
row = (ResultSetImpl) statement.executeQuery();
row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
row.close();
statement.close();
while (ct > 0) {
statement = connection.prepareStatement(sql);
int ls = pSize * num;
......@@ -91,20 +96,37 @@ public class DMUserMemberProcessor extends DataMigrationProcessorService {
userMemberList.add(userMember);
vos.add(vo);
}
}
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, userMemberList.size());
if (!CollectionUtils.isEmpty(userMemberList)) {
if (userMemberList.size() == 500) {
tl += userMemberList.size();
mongoTemplate.insert(vos, AdamUserMemberVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamUserMemberService.saveBatch(userMemberList));
vos.clear();
userMemberList.clear();
}
}
row.close();
statement.close();
num++;
ct -= pSize;
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, tl);
}
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
row.close();
statement.close();
connection.close();
} catch (Exception e) {
log.error("ex:getConnection,dG/dg:{}/{},msg:{}", dG, dg, e.getMessage());
}
try {
if (null != row) row.close();
} catch (SQLException ignored) {
}
try {
if (null != statement) statement.close();
} catch (SQLException ignored) {
}
try {
if (null != connection) connection.close();
} catch (SQLException ignored) {
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment