记得上下班打卡 | git大法好，push需谨慎

Skip to content

L
liquidnet-bus-v1
Commit a1485b47 authored by 张国柄's avatar 张国柄
Browse files
Options

opt dm;

parent c52c780b
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 829 additions and 632 deletions
liquidnet-bus-config/liquidnet-config/liquidnet-service-adam.yml
View file @ a1485b47
...@@ -118,21 +118,14 @@ spring: ...@@ -118,21 +118,14 @@ spring:
url: jdbc:mysql://${liquidnet.mysql.urlHostAndPort}/${liquidnet.mysql.database-name}?serverTimezone=Asia/Shanghai&characterEncoding=utf-8&useSSL=false url: jdbc:mysql://${liquidnet.mysql.urlHostAndPort}/${liquidnet.mysql.database-name}?serverTimezone=Asia/Shanghai&characterEncoding=utf-8&useSSL=false
username: ${liquidnet.mysql.username} username: ${liquidnet.mysql.username}
password: ${liquidnet.mysql.password} password: ${liquidnet.mysql.password}
# type: org.apache.tomcat.jdbc.pool.DataSource # type: org.apache.tomcat.jdbc.pool.DataSource
driver-class-name: com.mysql.cj.jdbc.Driver driver-class-name: com.mysql.cj.jdbc.Driver
filters: stat hikari:
maxActive: 200 maximum-pool-size: 500
initialSize: 1 # 迁移数据暂设10分钟
maxWait: 60000 connection-timeout: 600000
minIdle: 1 minimum-idle: 10
timeBetweenEvictionRunsMillis: 60000 connection-test-query: SELECT 1
minEvictableIdleTimeMillis: 300000
validationQuery: select 'x'
testWhileIdle: true
testOnBorrow: false
testOnReturn: false
poolPreparedStatements: true
maxOpenPreparedStatements: 20
# ----------------------------------------------------------- # -----------------------------------------------------------
# ----------------------------------------------------------- # -----------------------------------------------------------
......
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/docu/db_ln_adam_initialdata.sql
View file @ a1485b47
...@@ -42,7 +42,7 @@ create table adam_user ...@@ -42,7 +42,7 @@ create table adam_user
( (
mid bigint unsigned auto_increment primary key, mid bigint unsigned auto_increment primary key,
uid varchar(64) not null, uid varchar(64) not null,
mobile varchar(30) not null, mobile varchar(30),
passwd varchar(64), passwd varchar(64),
pay_code varchar(64), pay_code varchar(64),
state tinyint default 0 comment '1-NORMAL,2-INVALID', state tinyint default 0 comment '1-NORMAL,2-INVALID',
......
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/controller/admin/AdamDMAdminController.java
View file @ a1485b47
...@@ -67,6 +67,7 @@ public class AdamDMAdminController extends AdamBaseController { ...@@ -67,6 +67,7 @@ public class AdamDMAdminController extends AdamBaseController {
log.info("==================" + k); log.info("==================" + k);
dmUserInformationProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : ""); dmUserInformationProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : "");
}); });
try { Thread.sleep(100L); } catch (InterruptedException ignored) {}
} }
exec.shutdown(); exec.shutdown();
} else { } else {
...@@ -95,6 +96,7 @@ public class AdamDMAdminController extends AdamBaseController { ...@@ -95,6 +96,7 @@ public class AdamDMAdminController extends AdamBaseController {
log.info("==================" + k); log.info("==================" + k);
dmThirdPartsProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : ""); dmThirdPartsProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : "");
}); });
try { Thread.sleep(100L); } catch (InterruptedException ignored) {}
} }
exec.shutdown(); exec.shutdown();
} else { } else {
...@@ -123,6 +125,7 @@ public class AdamDMAdminController extends AdamBaseController { ...@@ -123,6 +125,7 @@ public class AdamDMAdminController extends AdamBaseController {
log.info("==================" + k); log.info("==================" + k);
dmRealNameProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : ""); dmRealNameProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : "");
}); });
try { Thread.sleep(100L); } catch (InterruptedException ignored) {}
} }
exec.shutdown(); exec.shutdown();
} else { } else {
...@@ -151,6 +154,7 @@ public class AdamDMAdminController extends AdamBaseController { ...@@ -151,6 +154,7 @@ public class AdamDMAdminController extends AdamBaseController {
log.info("==================" + k); log.info("==================" + k);
dmCollectionProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : ""); dmCollectionProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : "");
}); });
try { Thread.sleep(100L); } catch (InterruptedException ignored) {}
} }
exec.shutdown(); exec.shutdown();
} else { } else {
...@@ -179,6 +183,7 @@ public class AdamDMAdminController extends AdamBaseController { ...@@ -179,6 +183,7 @@ public class AdamDMAdminController extends AdamBaseController {
log.info("==================" + k); log.info("==================" + k);
dmEntersProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : ""); dmEntersProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : "");
}); });
try { Thread.sleep(100L); } catch (InterruptedException ignored) {}
} }
exec.shutdown(); exec.shutdown();
} else { } else {
...@@ -207,6 +212,7 @@ public class AdamDMAdminController extends AdamBaseController { ...@@ -207,6 +212,7 @@ public class AdamDMAdminController extends AdamBaseController {
log.info("==================" + k); log.info("==================" + k);
dmAddressesProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : ""); dmAddressesProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : "");
}); });
try { Thread.sleep(100L); } catch (InterruptedException ignored) {}
} }
exec.shutdown(); exec.shutdown();
} else { } else {
...@@ -235,6 +241,7 @@ public class AdamDMAdminController extends AdamBaseController { ...@@ -235,6 +241,7 @@ public class AdamDMAdminController extends AdamBaseController {
log.info("==================" + k); log.info("==================" + k);
dmUserMemberProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : ""); dmUserMemberProcessor.dataProcessing(k, dG, null, null, k == 0 ? "1" : "");
}); });
try { Thread.sleep(100L); } catch (InterruptedException ignored) {}
} }
exec.shutdown(); exec.shutdown();
} else { } else {
......
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMAddressesProcessor.java
View file @ a1485b47
...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired; ...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
...@@ -47,61 +47,82 @@ public class DMAddressesProcessor extends DataMigrationProcessorService { ...@@ -47,61 +47,82 @@ public class DMAddressesProcessor extends DataMigrationProcessorService {
} }
sql += " order by id limit ?,?"; sql += " order by id limit ?,?";
Class.forName(DB_DRIVER); Connection connection = null;
Connection connection = DriverManager.getConnection(DB_URL + DB_NAME_MALL, DB_USER, DB_PWD); PreparedStatement statement = null;
ResultSetImpl row = null;
PreparedStatement statement = connection.prepareStatement(sqlCount); try {
ResultSetImpl row = (ResultSetImpl) statement.executeQuery(); Class.forName(DB_DRIVER);
row.first(); connection = DriverManager.getConnection(DB_URL + DB_NAME_MALL, DB_USER, DB_PWD);
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct; statement = connection.prepareStatement(sqlCount);
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
while (ct > 0) {
statement = connection.prepareStatement(sql);
int ls = pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} - begin", ls, pSize); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
List<AdamAddresses> addressesList = new ArrayList<>(); row.close();
List<AdamAddressesVo> vos = new ArrayList<>(); statement.close();
while (row.next()) { while (ct > 0) {
String uid = row.getString(2); statement = connection.prepareStatement(sql);
int ls = pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} - begin", ls, pSize);
if (!reUidList.contains(uid)) { List<AdamAddresses> addressesList = new ArrayList<>();
AdamAddresses addresses = new AdamAddresses(); List<AdamAddressesVo> vos = new ArrayList<>();
addresses.setAddressesId(row.getString(1)); while (row.next()) {
addresses.setUid(uid); String uid = row.getString(2);
addresses.setName(row.getString(3)); if (!reUidList.contains(uid)) {
addresses.setPhone(row.getString(4)); AdamAddresses addresses = new AdamAddresses();
addresses.setProvince(row.getString(5)); addresses.setAddressesId(row.getString(1));
addresses.setCity(row.getString(6)); addresses.setUid(uid);
addresses.setCounty(row.getString(7)); addresses.setName(row.getString(3));
addresses.setAddress(row.getString(8)); addresses.setPhone(row.getString(4));
addresses.setIsDefault(row.getBoolean(9)); addresses.setProvince(row.getString(5));
addresses.setState(1); addresses.setCity(row.getString(6));
addresses.setCreatedAt(row.getLocalDateTime(10)); addresses.setCounty(row.getString(7));
addresses.setAddress(row.getString(8));
addresses.setIsDefault(row.getBoolean(9));
addresses.setState(1);
addresses.setCreatedAt(row.getLocalDateTime(10));
AdamAddressesVo vo = AdamAddressesVo.getNew().copy(addresses); AdamAddressesVo vo = AdamAddressesVo.getNew().copy(addresses);
addressesList.add(addresses); addressesList.add(addresses);
vos.add(vo); vos.add(vo);
}
if (addressesList.size() == 500) {
tl += addressesList.size();
mongoTemplate.insert(vos, AdamAddressesVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamAddressesService.saveBatch(addressesList));
vos.clear();
addressesList.clear();
}
} }
} row.close();
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, addressesList.size()); statement.close();
if (!CollectionUtils.isEmpty(addressesList)) {
tl += addressesList.size(); num++;
mongoTemplate.insert(vos, AdamAddressesVo.class.getSimpleName()); ct -= pSize;
log.info("DM.execute.limit.result:{}", adamAddressesService.saveBatch(addressesList));
}
num++; log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, tl);
ct -= pSize; }
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
} catch (Exception e) {
log.error("ex:getConnection,dG/dg:{}/{},msg:{}", dG, dg, e.getMessage());
}
try {
if (null != row) row.close();
} catch (SQLException ignored) {
}
try {
if (null != statement) statement.close();
} catch (SQLException ignored) {
}
try {
if (null != connection) connection.close();
} catch (SQLException ignored) {
} }
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
row.close();
statement.close();
connection.close();
} }
} }
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMCollectionProcessor.java
View file @ a1485b47
...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired; ...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
...@@ -35,7 +35,7 @@ public class DMCollectionProcessor extends DataMigrationProcessorService { ...@@ -35,7 +35,7 @@ public class DMCollectionProcessor extends DataMigrationProcessorService {
log.info("DM.flush.AdamCollectVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").exists(true)), AdamCollectVo.class.getSimpleName()).getDeletedCount()); log.info("DM.flush.AdamCollectVo:{}", mongoTemplate.remove(Query.query(Criteria.where("_id").exists(true)), AdamCollectVo.class.getSimpleName()).getDeletedCount());
} }
String sqlCount = "select count(1) from user_collections where type='TICKET' and status=1 and content_id>=5722 and created_at"+(StringUtils.isBlank(incrDt) ? "<" : ">=")+"curdate()"; String sqlCount = "select count(1) from user_collections where type='TICKET' and status=1 and content_id>=5722 and created_at" + (StringUtils.isBlank(incrDt) ? "<" : ">=") + "curdate()";
if (null != dg) { if (null != dg) {
sqlCount = sqlCount + " and id%" + dG + "=" + dg; sqlCount = sqlCount + " and id%" + dG + "=" + dg;
} }
...@@ -48,53 +48,75 @@ public class DMCollectionProcessor extends DataMigrationProcessorService { ...@@ -48,53 +48,75 @@ public class DMCollectionProcessor extends DataMigrationProcessorService {
} }
sql += " order by id limit ?,?"; sql += " order by id limit ?,?";
Class.forName(DB_DRIVER); Connection connection = null;
Connection connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD); PreparedStatement statement = null;
ResultSetImpl row = null;
PreparedStatement statement = connection.prepareStatement(sqlCount); try {
ResultSetImpl row = (ResultSetImpl) statement.executeQuery(); Class.forName(DB_DRIVER);
row.first(); connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD);
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct; statement = connection.prepareStatement(sqlCount);
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
while (ct > 0) {
statement = connection.prepareStatement(sql);
int ls = pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} - begin", ls, pSize); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
row.close();
statement.close();
while (ct > 0) {
statement = connection.prepareStatement(sql);
int ls = pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} - begin", ls, pSize);
List<AdamCollection> collectionList = new ArrayList<>(); List<AdamCollection> collectionList = new ArrayList<>();
List<AdamCollectVo> vos = new ArrayList<>(); List<AdamCollectVo> vos = new ArrayList<>();
while (row.next()) { while (row.next()) {
String uid = row.getString(1); String uid = row.getString(1);
if (!reUidList.contains(uid)) { if (!reUidList.contains(uid)) {
AdamCollection collection = new AdamCollection(); AdamCollection collection = new AdamCollection();
collection.setUid(uid); collection.setUid(uid);
collection.setContentId(row.getString(2)); collection.setContentId(row.getString(2));
collection.setType(row.getString(3)); collection.setType(row.getString(3));
collection.setState(row.getInt(4)); collection.setState(row.getInt(4));
collection.setCreatedAt(row.getLocalDateTime(5)); collection.setCreatedAt(row.getLocalDateTime(5));
collection.setUpdatedAt(row.getLocalDateTime(6)); collection.setUpdatedAt(row.getLocalDateTime(6));
collectionList.add(collection); collectionList.add(collection);
vos.add(AdamCollectVo.getNew().copy(collection)); vos.add(AdamCollectVo.getNew().copy(collection));
}
if (collectionList.size() == 500) {
tl += collectionList.size();
mongoTemplate.insert(vos, AdamCollectVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamCollectionService.saveBatch(collectionList));
vos.clear();
collectionList.clear();
}
} }
} row.close();
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, collectionList.size()); statement.close();
if (!CollectionUtils.isEmpty(collectionList)) {
tl += collectionList.size();
mongoTemplate.insert(vos, AdamCollectVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamCollectionService.saveBatch(collectionList));
}
num++; num++;
ct -= pSize; ct -= pSize;
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, tl);
}
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
} catch (Exception e) {
log.error("ex:getConnection,dG/dg:{}/{},msg:{}", dG, dg, e.getMessage());
}
try {
if (null != row) row.close();
} catch (SQLException ignored) {
}
try {
if (null != statement) statement.close();
} catch (SQLException ignored) {
}
try {
if (null != connection) connection.close();
} catch (SQLException ignored) {
} }
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
row.close();
statement.close();
connection.close();
} }
} }
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMEntersProcessor.java
View file @ a1485b47
...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired; ...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
...@@ -48,59 +48,80 @@ public class DMEntersProcessor extends DataMigrationProcessorService { ...@@ -48,59 +48,80 @@ public class DMEntersProcessor extends DataMigrationProcessorService {
} }
sql += " order by id limit ?,?"; sql += " order by id limit ?,?";
Class.forName(DB_DRIVER); Connection connection = null;
Connection connection = DriverManager.getConnection(DB_URL + DB_NAME_MALL, DB_USER, DB_PWD); PreparedStatement statement = null;
ResultSetImpl row = null;
PreparedStatement statement = connection.prepareStatement(sqlCount); try {
ResultSetImpl row = (ResultSetImpl) statement.executeQuery(); Class.forName(DB_DRIVER);
row.first(); connection = DriverManager.getConnection(DB_URL + DB_NAME_MALL, DB_USER, DB_PWD);
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct; statement = connection.prepareStatement(sqlCount);
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
while (ct > 0) {
statement = connection.prepareStatement(sql);
int ls = pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} - begin", ls, pSize); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
List<AdamEnters> entersList = new ArrayList<>(); row.close();
List<AdamEntersVo> vos = new ArrayList<>(); statement.close();
while (row.next()) { while (ct > 0) {
String uid = row.getString(2); statement = connection.prepareStatement(sql);
int ls = pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} - begin", ls, pSize);
if (!reUidList.contains(uid)) { List<AdamEnters> entersList = new ArrayList<>();
AdamEnters enters = new AdamEnters(); List<AdamEntersVo> vos = new ArrayList<>();
enters.setEntersId(row.getString(1)); while (row.next()) {
enters.setUid(uid); String uid = row.getString(2);
enters.setType(row.getInt(3)); if (!reUidList.contains(uid)) {
enters.setName(row.getString(4)); AdamEnters enters = new AdamEnters();
enters.setMobile(row.getString(5)); enters.setEntersId(row.getString(1));
enters.setIdCard(row.getString(6)); enters.setUid(uid);
enters.setIsDefault(row.getBoolean(7)); enters.setType(row.getInt(3));
enters.setState(1); enters.setName(row.getString(4));
enters.setCreatedAt(row.getLocalDateTime(8)); enters.setMobile(row.getString(5));
enters.setIdCard(row.getString(6));
enters.setIsDefault(row.getBoolean(7));
enters.setState(1);
enters.setCreatedAt(row.getLocalDateTime(8));
AdamEntersVo vo = AdamEntersVo.getNew().copy(enters); AdamEntersVo vo = AdamEntersVo.getNew().copy(enters);
entersList.add(enters); entersList.add(enters);
vos.add(vo); vos.add(vo);
}
if (entersList.size() == 500) {
tl += entersList.size();
mongoTemplate.insert(vos, AdamEntersVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamEntersService.saveBatch(entersList));
vos.clear();
entersList.clear();
}
} }
} row.close();
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, entersList.size()); statement.close();
if (!CollectionUtils.isEmpty(entersList)) {
tl += entersList.size(); num++;
mongoTemplate.insert(vos, AdamEntersVo.class.getSimpleName()); ct -= pSize;
log.info("DM.execute.limit.result:{}", adamEntersService.saveBatch(entersList));
}
num++; log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, tl);
ct -= pSize; }
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
} catch (Exception e) {
log.error("ex:getConnection,dG/dg:{}/{},msg:{}", dG, dg, e.getMessage());
}
try {
if (null != row) row.close();
} catch (SQLException ignored) {
}
try {
if (null != statement) statement.close();
} catch (SQLException ignored) {
}
try {
if (null != connection) connection.close();
} catch (SQLException ignored) {
} }
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
row.close();
statement.close();
connection.close();
} }
} }
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMMemberOrderProcessor.java
View file @ a1485b47
...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired; ...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
...@@ -73,71 +73,93 @@ public class DMMemberOrderProcessor extends DataMigrationProcessorService { ...@@ -73,71 +73,93 @@ public class DMMemberOrderProcessor extends DataMigrationProcessorService {
} }
sql += " order by id limit ?,?"; sql += " order by id limit ?,?";
Class.forName(DB_DRIVER); Connection connection = null;
Connection connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD); PreparedStatement statement = null;
ResultSetImpl row = null;
PreparedStatement statement = connection.prepareStatement(sqlCount); try {
ResultSetImpl row = (ResultSetImpl) statement.executeQuery(); Class.forName(DB_DRIVER);
row.first(); connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD);
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct; statement = connection.prepareStatement(sqlCount);
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
while (ct > 0) {
statement = connection.prepareStatement(sql);
int ls = pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} - begin", ls, pSize); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
List<AdamMemberOrder> memberOrderList = new ArrayList<>(); log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
List<AdamMemberOrderVo> vos = new ArrayList<>();
while (row.next()) { row.close();
String uid = row.getString(2); statement.close();
if (!reUidList.contains(uid)) { while (ct > 0) {
AdamMemberOrder memberOrder = new AdamMemberOrder(); statement = connection.prepareStatement(sql);
memberOrder.setOrderNo(row.getString(1)); int ls = pSize * num;
memberOrder.setUid(uid); statement.setInt(1, ls);
memberOrder.setMode(row.getInt(3)); statement.setInt(2, pSize);
memberOrder.setPrice(row.getBigDecimal(4)); row = (ResultSetImpl) statement.executeQuery();
memberOrder.setPricePaid(row.getBigDecimal(5)); log.info("DM.execute.limit {},{} - begin", ls, pSize);
memberOrder.setMemberName(MEMBER_NAME);
memberOrder.setMemberId(MEMBER_ID); List<AdamMemberOrder> memberOrderList = new ArrayList<>();
memberOrder.setMemberPriceId(MEMBER_ID); List<AdamMemberOrderVo> vos = new ArrayList<>();
memberOrder.setDays(365); while (row.next()) {
memberOrder.setState(row.getInt(6)); String uid = row.getString(2);
memberOrder.setMemberNo(row.getString(7)); if (!reUidList.contains(uid)) {
memberOrder.setBirthday(row.getString(8)); AdamMemberOrder memberOrder = new AdamMemberOrder();
String payType = row.getString(9); memberOrder.setOrderNo(row.getString(1));
memberOrder.setDeviceFrom(dfMap.get(payType)); memberOrder.setUid(uid);
memberOrder.setPayType(ptMap.get(payType)); memberOrder.setMode(row.getInt(3));
memberOrder.setPayNo(row.getString(10)); memberOrder.setPrice(row.getBigDecimal(4));
memberOrder.setPaymentAt(row.getLocalDateTime(11)); memberOrder.setPricePaid(row.getBigDecimal(5));
memberOrder.setCreatedAt(row.getLocalDateTime(12)); memberOrder.setMemberName(MEMBER_NAME);
memberOrder.setUpdatedAt(row.getLocalDateTime(13)); memberOrder.setMemberId(MEMBER_ID);
memberOrder.setClientIp(row.getString(14)); memberOrder.setMemberPriceId(MEMBER_ID);
memberOrder.setSource(row.getString(15)); memberOrder.setDays(365);
memberOrder.setVersion(row.getString(16)); memberOrder.setState(row.getInt(6));
memberOrder.setMemberNo(row.getString(7));
AdamMemberOrderVo vo = AdamMemberOrderVo.getNew().copy(memberOrder); memberOrder.setBirthday(row.getString(8));
String payType = row.getString(9);
memberOrderList.add(memberOrder); memberOrder.setDeviceFrom(dfMap.get(payType));
vos.add(vo); memberOrder.setPayType(ptMap.get(payType));
memberOrder.setPayNo(row.getString(10));
memberOrder.setPaymentAt(row.getLocalDateTime(11));
memberOrder.setCreatedAt(row.getLocalDateTime(12));
memberOrder.setUpdatedAt(row.getLocalDateTime(13));
memberOrder.setClientIp(row.getString(14));
memberOrder.setSource(row.getString(15));
memberOrder.setVersion(row.getString(16));
AdamMemberOrderVo vo = AdamMemberOrderVo.getNew().copy(memberOrder);
memberOrderList.add(memberOrder);
vos.add(vo);
}
if (memberOrderList.size() == 500) {
tl += memberOrderList.size();
mongoTemplate.insert(vos, AdamMemberOrderVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamMemberOrderService.saveBatch(memberOrderList));
vos.clear();
memberOrderList.clear();
}
} }
} row.close();
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, memberOrderList.size()); statement.close();
if (!CollectionUtils.isEmpty(memberOrderList)) {
tl += memberOrderList.size(); num++;
mongoTemplate.insert(vos, AdamMemberOrderVo.class.getSimpleName()); ct -= pSize;
log.info("DM.execute.limit.result:{}", adamMemberOrderService.saveBatch(memberOrderList));
}
num++; log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, tl);
ct -= pSize; }
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
} catch (Exception e) {
log.error("ex:getConnection,dG/dg:{}/{},msg:{}", dG, dg, e.getMessage());
}
try {
if (null != row) row.close();
} catch (SQLException ignored) {
}
try {
if (null != statement) statement.close();
} catch (SQLException ignored) {
}
try {
if (null != connection) connection.close();
} catch (SQLException ignored) {
} }
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
row.close();
statement.close();
connection.close();
} }
} }
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMRealNameProcessor.java
View file @ a1485b47
...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired; ...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
...@@ -48,62 +48,84 @@ public class DMRealNameProcessor extends DataMigrationProcessorService { ...@@ -48,62 +48,84 @@ public class DMRealNameProcessor extends DataMigrationProcessorService {
} }
sql += " order by id limit ?,?"; sql += " order by id limit ?,?";
Class.forName(DB_DRIVER); Connection connection = null;
Connection connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD); PreparedStatement statement = null;
ResultSetImpl row = null;
PreparedStatement statement = connection.prepareStatement(sqlCount); try {
ResultSetImpl row = (ResultSetImpl) statement.executeQuery(); Class.forName(DB_DRIVER);
row.first(); connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD);
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct; statement = connection.prepareStatement(sqlCount);
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
while (ct > 0) {
statement = connection.prepareStatement(sql);
int ls = pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} - begin", ls, pSize); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
row.close();
statement.close();
while (ct > 0) {
statement = connection.prepareStatement(sql);
int ls = pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} - begin", ls, pSize);
List<AdamRealName> realNameList = new ArrayList<>(); List<AdamRealName> realNameList = new ArrayList<>();
List<AdamRealInfoVo> vos = new ArrayList<>(); List<AdamRealInfoVo> vos = new ArrayList<>();
while (row.next()) { while (row.next()) {
String uid = row.getString(2); String uid = row.getString(2);
if (!reUidList.contains(uid)) { if (!reUidList.contains(uid)) {
AdamRealName realName = new AdamRealName(); AdamRealName realName = new AdamRealName();
realName.setRealNameId(row.getString(1)); realName.setRealNameId(row.getString(1));
realName.setUid(uid); realName.setUid(uid);
realName.setType("1"); realName.setType("1");
realName.setName(row.getString(3)); realName.setName(row.getString(3));
realName.setIdCard(row.getString(4)); realName.setIdCard(row.getString(4));
realName.setState(1); realName.setState(1);
realName.setCreatedAt(row.getLocalDateTime(5)); realName.setCreatedAt(row.getLocalDateTime(5));
realName.setUpdatedAt(row.getLocalDateTime(6)); realName.setUpdatedAt(row.getLocalDateTime(6));
AdamRealInfoVo vo = AdamRealInfoVo.getNew(); AdamRealInfoVo vo = AdamRealInfoVo.getNew();
vo.setUid(uid); vo.setUid(uid);
vo.setType(realName.getType()); vo.setType(realName.getType());
vo.setName(realName.getName()); vo.setName(realName.getName());
vo.setIdCard(realName.getIdCard()); vo.setIdCard(realName.getIdCard());
vo.setState(1); vo.setState(1);
realNameList.add(realName); realNameList.add(realName);
vos.add(vo); vos.add(vo);
}
if (realNameList.size() == 500) {
tl += realNameList.size();
mongoTemplate.insert(vos, AdamRealInfoVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamRealNameService.saveBatch(realNameList));
vos.clear();
realNameList.clear();
}
} }
} row.close();
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, realNameList.size()); statement.close();
if (!CollectionUtils.isEmpty(realNameList)) {
tl += realNameList.size();
mongoTemplate.insert(vos, AdamRealInfoVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamRealNameService.saveBatch(realNameList));
}
num++; num++;
ct -= pSize; ct -= pSize;
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, tl);
}
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
} catch (Exception e) {
log.error("ex:getConnection,dG/dg:{}/{},msg:{}", dG, dg, e.getMessage());
}
try {
if (null != row) row.close();
} catch (SQLException ignored) {
}
try {
if (null != statement) statement.close();
} catch (SQLException ignored) {
}
try {
if (null != connection) connection.close();
} catch (SQLException ignored) {
} }
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
row.close();
statement.close();
connection.close();
} }
} }
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMThirdPartsProcessor.java
View file @ a1485b47
...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired; ...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
...@@ -36,7 +36,7 @@ public class DMThirdPartsProcessor extends DataMigrationProcessorService { ...@@ -36,7 +36,7 @@ public class DMThirdPartsProcessor extends DataMigrationProcessorService {
} }
String sqlCount = "select count(1) from (\n" + String sqlCount = "select count(1) from (\n" +
" select row_number() over (partition by uid,type order by created_at desc) rn,id,uid,open_id,avatar,nickname,type,created_at,updated_at from user_third_parts where `status`=1 and delete_tag in (0,1) and open_id<>'' and uid>0 and created_at"+(StringUtils.isBlank(incrDt) ? "<" : ">=")+"curdate()\n" + " select row_number() over (partition by uid,type order by created_at desc) rn,id,uid,open_id,avatar,nickname,type,created_at,updated_at from user_third_parts where `status`=1 and delete_tag in (0,1) and open_id<>'' and uid>0 and created_at" + (StringUtils.isBlank(incrDt) ? "<" : ">=") + "curdate()\n" +
" ) t where t.rn=1"; " ) t where t.rn=1";
if (null != dg) { if (null != dg) {
sqlCount = sqlCount + " and id%" + dG + "=" + dg; sqlCount = sqlCount + " and id%" + dG + "=" + dg;
...@@ -50,65 +50,87 @@ public class DMThirdPartsProcessor extends DataMigrationProcessorService { ...@@ -50,65 +50,87 @@ public class DMThirdPartsProcessor extends DataMigrationProcessorService {
} }
sql += " order by id limit ?,?"; sql += " order by id limit ?,?";
Class.forName(DB_DRIVER); Connection connection = null;
Connection connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD); PreparedStatement statement = null;
ResultSetImpl row = null;
PreparedStatement statement = connection.prepareStatement(sqlCount); try {
ResultSetImpl row = (ResultSetImpl) statement.executeQuery(); Class.forName(DB_DRIVER);
row.first(); connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD);
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct; statement = connection.prepareStatement(sqlCount);
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
while (ct > 0) {
statement = connection.prepareStatement(sql);
int ls = pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} - begin", ls, pSize); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
row.close();
statement.close();
while (ct > 0) {
statement = connection.prepareStatement(sql);
int ls = pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} - begin", ls, pSize);
List<AdamThirdParty> thirdPartyList = new ArrayList<>(); List<AdamThirdParty> thirdPartyList = new ArrayList<>();
List<AdamThirdPartInfoVo> vos = new ArrayList<>(); List<AdamThirdPartInfoVo> vos = new ArrayList<>();
while (row.next()) { while (row.next()) {
String uid = row.getString(1); String uid = row.getString(1);
if (!reUidList.contains(uid)) { if (!reUidList.contains(uid)) {
AdamThirdParty thirdParty = new AdamThirdParty(); AdamThirdParty thirdParty = new AdamThirdParty();
thirdParty.setUid(uid); thirdParty.setUid(uid);
thirdParty.setOpenId(row.getString(2)); thirdParty.setOpenId(row.getString(2));
thirdParty.setAvatar(row.getString(3)); thirdParty.setAvatar(row.getString(3));
thirdParty.setNickname(row.getString(4)); thirdParty.setNickname(row.getString(4));
thirdParty.setPlatform(row.getString(5)); thirdParty.setPlatform(row.getString(5));
thirdParty.setState(1); thirdParty.setState(1);
thirdParty.setCreatedAt(row.getLocalDateTime(6)); thirdParty.setCreatedAt(row.getLocalDateTime(6));
thirdParty.setUpdatedAt(row.getLocalDateTime(7)); thirdParty.setUpdatedAt(row.getLocalDateTime(7));
AdamThirdPartInfoVo vo = AdamThirdPartInfoVo.getNew(); AdamThirdPartInfoVo vo = AdamThirdPartInfoVo.getNew();
vo.setUid(thirdParty.getUid()); vo.setUid(thirdParty.getUid());
vo.setOpenId(thirdParty.getOpenId()); vo.setOpenId(thirdParty.getOpenId());
vo.setNickname(thirdParty.getNickname()); vo.setNickname(thirdParty.getNickname());
vo.setAvatar(thirdParty.getAvatar()); vo.setAvatar(thirdParty.getAvatar());
vo.setPlatform(thirdParty.getPlatform()); vo.setPlatform(thirdParty.getPlatform());
vo.setState(thirdParty.getState()); vo.setState(thirdParty.getState());
vo.setCreatedAt(thirdParty.getCreatedAt()); vo.setCreatedAt(thirdParty.getCreatedAt());
vo.setUpdatedAt(thirdParty.getUpdatedAt()); vo.setUpdatedAt(thirdParty.getUpdatedAt());
thirdPartyList.add(thirdParty); thirdPartyList.add(thirdParty);
vos.add(vo); vos.add(vo);
}
if (thirdPartyList.size() == 500) {
tl += thirdPartyList.size();
mongoTemplate.insert(vos, AdamThirdPartInfoVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamThirdPartyService.saveBatch(thirdPartyList));
vos.clear();
thirdPartyList.clear();
}
} }
} row.close();
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, thirdPartyList.size()); statement.close();
if (!CollectionUtils.isEmpty(thirdPartyList)) {
tl += thirdPartyList.size();
mongoTemplate.insert(vos, AdamThirdPartInfoVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamThirdPartyService.saveBatch(thirdPartyList));
}
num++; num++;
ct -= pSize; ct -= pSize;
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, tl);
}
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
} catch (Exception e) {
log.error("ex:getConnection,dG/dg:{}/{},msg:{}", dG, dg, e.getMessage());
}
try {
if (null != row) row.close();
} catch (SQLException ignored) {
}
try {
if (null != statement) statement.close();
} catch (SQLException ignored) {
}
try {
if (null != connection) connection.close();
} catch (SQLException ignored) {
} }
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
row.close();
statement.close();
connection.close();
} }
} }
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMUserInformationProcessor.java
View file @ a1485b47
...@@ -18,11 +18,11 @@ import org.springframework.beans.factory.annotation.Autowired; ...@@ -18,11 +18,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
...@@ -68,85 +68,107 @@ public class DMUserInformationProcessor extends DataMigrationProcessorService { ...@@ -68,85 +68,107 @@ public class DMUserInformationProcessor extends DataMigrationProcessorService {
} }
sql += " order by uid limit ?,?"; sql += " order by uid limit ?,?";
Class.forName(DB_DRIVER); Connection connection = null;
Connection connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD); PreparedStatement statement = null;
ResultSetImpl row = null;
PreparedStatement statement = connection.prepareStatement(sqlCount); try {
ResultSetImpl row = (ResultSetImpl) statement.executeQuery(); Class.forName(DB_DRIVER);
row.first(); connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD);
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct; statement = connection.prepareStatement(sqlCount);
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
while (ct > 0) {
statement = connection.prepareStatement(sql);
int ls = pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} - begin", ls, pSize); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
List<AdamUser> userList = new ArrayList<>(); log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
List<AdamUserInfo> userInfoList = new ArrayList<>();
List<AdamUserInfoVo> vos = new ArrayList<>(); row.close();
while (row.next()) { statement.close();
AdamTagVo sex = sexMap.get(row.getString(4)); while (ct > 0) {
String uid = row.getString(1); statement = connection.prepareStatement(sql);
if (!reUidList.contains(uid)) { int ls = pSize * num;
AdamUser user = new AdamUser(); statement.setInt(1, ls);
user.setUid(uid); statement.setInt(2, pSize);
user.setMobile(row.getString(2)); row = (ResultSetImpl) statement.executeQuery();
user.setCreatedAt(row.getLocalDateTime(10)); log.info("DM.execute.limit {},{} - begin", ls, pSize);
user.setUpdatedAt(row.getLocalDateTime(11));
user.setState(1); List<AdamUser> userList = new ArrayList<>();
List<AdamUserInfo> userInfoList = new ArrayList<>();
AdamUserInfo userInfo = new AdamUserInfo(); List<AdamUserInfoVo> vos = new ArrayList<>();
userInfo.setUid(user.getUid()); while (row.next()) {
userInfo.setNickname(row.getString(3)); AdamTagVo sex = sexMap.get(row.getString(4));
userInfo.setSex(JsonUtils.toJson(sex)); String uid = row.getString(1);
userInfo.setBirthday(row.getLocalDate(5)); if (!reUidList.contains(uid)) {
userInfo.setArea(row.getString(6)); AdamUser user = new AdamUser();
userInfo.setSignature(row.getString(7)); user.setUid(uid);
userInfo.setAvatar(row.getString(8)); user.setMobile(row.getString(2));
userInfo.setBackground(row.getString(9)); user.setCreatedAt(row.getLocalDateTime(10));
userInfo.setQrCode("lN".concat(userInfo.getUid()).concat(RandomStringUtils.randomAlphanumeric(5).toUpperCase())); user.setUpdatedAt(row.getLocalDateTime(11));
userInfo.setRongCloudToken(row.getString(13)); user.setState(1);
userInfo.setRongCloudTag(row.getInt(14));
AdamUserInfo userInfo = new AdamUserInfo();
AdamUserInfoVo vo = AdamUserInfoVo.getNew(); userInfo.setUid(user.getUid());
vo.setUid(user.getUid()); userInfo.setNickname(row.getString(3));
vo.setMobile(user.getMobile()); userInfo.setSex(JsonUtils.toJson(sex));
vo.setNickname(userInfo.getNickname()); userInfo.setBirthday(row.getLocalDate(5));
vo.setState(user.getState()); userInfo.setArea(row.getString(6));
vo.setSex(sex); userInfo.setSignature(row.getString(7));
vo.setBirthday(DateUtil.Formatter.yyyy_MM_dd.format(userInfo.getBirthday())); userInfo.setAvatar(row.getString(8));
vo.setArea(userInfo.getArea()); userInfo.setBackground(row.getString(9));
vo.setSignature(userInfo.getSignature()); userInfo.setQrCode("lN".concat(userInfo.getUid()).concat(RandomStringUtils.randomAlphanumeric(5).toUpperCase()));
vo.setAvatar(userInfo.getAvatar()); userInfo.setRongCloudToken(row.getString(13));
vo.setBackground(userInfo.getBackground()); userInfo.setRongCloudTag(row.getInt(14));
vo.setCreateAt(user.getCreatedAt());
vo.setUpdatedAt(user.getUpdatedAt()); AdamUserInfoVo vo = AdamUserInfoVo.getNew();
vo.setIsComplete(row.getInt(12)); vo.setUid(user.getUid());
vo.setQrCode(userInfo.getQrCode()); vo.setMobile(user.getMobile());
vo.setNickname(userInfo.getNickname());
userList.add(user); vo.setState(user.getState());
userInfoList.add(userInfo); vo.setSex(sex);
vos.add(vo); vo.setBirthday(DateUtil.Formatter.yyyy_MM_dd.format(userInfo.getBirthday()));
vo.setArea(userInfo.getArea());
vo.setSignature(userInfo.getSignature());
vo.setAvatar(userInfo.getAvatar());
vo.setBackground(userInfo.getBackground());
vo.setCreateAt(user.getCreatedAt());
vo.setUpdatedAt(user.getUpdatedAt());
vo.setIsComplete(row.getInt(12));
vo.setQrCode(userInfo.getQrCode());
userList.add(user);
userInfoList.add(userInfo);
vos.add(vo);
}
if (userList.size() == 500) {
tl += userList.size();
mongoTemplate.insert(vos, AdamUserInfoVo.class.getSimpleName());
log.info("DM.execute.result:{} - {}", adamUserService.saveBatch(userList), adamUserInfoService.saveBatch(userInfoList));
vos.clear();
userList.clear();
userInfoList.clear();
}
} }
} row.close();
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, userList.size()); statement.close();
if (!CollectionUtils.isEmpty(userList)) {
tl += userList.size(); num++;
mongoTemplate.insert(vos, AdamUserInfoVo.class.getSimpleName()); ct -= pSize;
log.info("DM.execute.limit.result:{}", adamUserService.saveBatch(userList));
log.info("DM.execute.limit.result:{}", adamUserInfoService.saveBatch(userInfoList));
}
num++; log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, tl);
ct -= pSize; }
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
} catch (Exception e) {
log.error("ex:getConnection,dG/dg:{}/{},msg:{}", dG, dg, e.getMessage());
}
try {
if (null != row) row.close();
} catch (SQLException ignored) {
}
try {
if (null != statement) statement.close();
} catch (SQLException ignored) {
}
try {
if (null != connection) connection.close();
} catch (SQLException ignored) {
} }
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
row.close();
statement.close();
connection.close();
} }
} }
liquidnet-bus-service/liquidnet-service-adam/liquidnet-service-adam-impl/src/main/java/com/liquidnet/service/adam/service/processor/DMUserMemberProcessor.java
View file @ a1485b47
...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired; ...@@ -12,11 +12,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
...@@ -48,63 +48,85 @@ public class DMUserMemberProcessor extends DataMigrationProcessorService { ...@@ -48,63 +48,85 @@ public class DMUserMemberProcessor extends DataMigrationProcessorService {
} }
sql += " order by id limit ?,?"; sql += " order by id limit ?,?";
Class.forName(DB_DRIVER); Connection connection = null;
Connection connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD); PreparedStatement statement = null;
ResultSetImpl row = null;
PreparedStatement statement = connection.prepareStatement(sqlCount); try {
ResultSetImpl row = (ResultSetImpl) statement.executeQuery(); Class.forName(DB_DRIVER);
row.first(); connection = DriverManager.getConnection(DB_URL + DB_NAME_PASSPORT, DB_USER, DB_PWD);
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct; statement = connection.prepareStatement(sqlCount);
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
while (ct > 0) {
statement = connection.prepareStatement(sql);
int ls = pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery(); row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} - begin", ls, pSize); row.first();
int ct = row.getInt(1), pSize = 1000, num = 0, tl = 0, pl = ct;
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> start.sql:{}", pl, num, pSize, tl, sql);
row.close();
statement.close();
while (ct > 0) {
statement = connection.prepareStatement(sql);
int ls = pSize * num;
statement.setInt(1, ls);
statement.setInt(2, pSize);
row = (ResultSetImpl) statement.executeQuery();
log.info("DM.execute.limit {},{} - begin", ls, pSize);
List<AdamUserMember> userMemberList = new ArrayList<>(); List<AdamUserMember> userMemberList = new ArrayList<>();
List<AdamUserMemberVo> vos = new ArrayList<>(); List<AdamUserMemberVo> vos = new ArrayList<>();
while (row.next()) { while (row.next()) {
String uid = row.getString(1); String uid = row.getString(1);
if (!reUidList.contains(uid)) { if (!reUidList.contains(uid)) {
AdamUserMember userMember = new AdamUserMember(); AdamUserMember userMember = new AdamUserMember();
userMember.setUid(uid); userMember.setUid(uid);
userMember.setMemberId(row.getString(2)); userMember.setMemberId(row.getString(2));
userMember.setMemberNo(row.getString(3)); userMember.setMemberNo(row.getString(3));
userMember.setState(row.getInt(4)); userMember.setState(row.getInt(4));
userMember.setExpiryAt(row.getLocalDateTime(5)); userMember.setExpiryAt(row.getLocalDateTime(5));
userMember.setCreatedAt(row.getLocalDateTime(6)); userMember.setCreatedAt(row.getLocalDateTime(6));
userMember.setUpdatedAt(row.getLocalDateTime(7)); userMember.setUpdatedAt(row.getLocalDateTime(7));
AdamUserMemberVo vo = AdamUserMemberVo.getNew(); AdamUserMemberVo vo = AdamUserMemberVo.getNew();
vo.setUid(uid); vo.setUid(uid);
vo.setMemberId(userMember.getMemberId()); vo.setMemberId(userMember.getMemberId());
vo.setMemberNo(userMember.getMemberNo()); vo.setMemberNo(userMember.getMemberNo());
vo.setState(userMember.getState()); vo.setState(userMember.getState());
vo.setExpiryAt(userMember.getExpiryAt()); vo.setExpiryAt(userMember.getExpiryAt());
vo.setCreatedAt(userMember.getCreatedAt()); vo.setCreatedAt(userMember.getCreatedAt());
vo.setUpdatedAt(userMember.getUpdatedAt()); vo.setUpdatedAt(userMember.getUpdatedAt());
userMemberList.add(userMember); userMemberList.add(userMember);
vos.add(vo); vos.add(vo);
}
if (userMemberList.size() == 500) {
tl += userMemberList.size();
mongoTemplate.insert(vos, AdamUserMemberVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamUserMemberService.saveBatch(userMemberList));
vos.clear();
userMemberList.clear();
}
} }
} row.close();
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, userMemberList.size()); statement.close();
if (!CollectionUtils.isEmpty(userMemberList)) {
tl += userMemberList.size();
mongoTemplate.insert(vos, AdamUserMemberVo.class.getSimpleName());
log.info("DM.execute.limit.result:{}", adamUserMemberService.saveBatch(userMemberList));
}
num++; num++;
ct -= pSize; ct -= pSize;
log.info("DM.execute.limit {},{} - handle.counts:{}", ls, pSize, tl);
}
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
} catch (Exception e) {
log.error("ex:getConnection,dG/dg:{}/{},msg:{}", dG, dg, e.getMessage());
}
try {
if (null != row) row.close();
} catch (SQLException ignored) {
}
try {
if (null != statement) statement.close();
} catch (SQLException ignored) {
}
try {
if (null != connection) connection.close();
} catch (SQLException ignored) {
} }
log.info("DM.execute.counts:{}/{},{} ~ {} ----------------------------> end", pl, num, pSize, tl);
row.close();
statement.close();
connection.close();
} }
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment