Guwan
/
Guwan-smartSchool
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

v2.0.15 

1.优化车辆图片并发高，用redis缓存，定时器消费处理
This commit is contained in:
gaoben 2021-07-13 15:06:53 +08:00
parent 52866c212e
commit 0a27d569e3
8 changed files with 309 additions and 134 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

188
shapelight-admin/src/main/java/net/shapelight/modules/car/controller/CarOpenApi.java
View File

@ -25,6 +25,7 @@ import net.shapelight.common.config.MinioConfig;
import net.shapelight.common.utils.*;
import net.shapelight.modules.ten.entity.*;
import net.shapelight.modules.ten.service.*;
import net.shapelight.modules.vo.CarImageCache;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@ -70,6 +71,9 @@ public class CarOpenApi {
@Autowired
private TenPackChannalService tenPackChannalService;
@Autowired
private RedisUtils redisUtils;
@PostMapping({"/getMsg/{cellId}"})
@ApiOperation("获取MSG")
@ApiImplicitParams({@ApiImplicitParam(name = "sn", value = "设备sn", paramType = "query", dataType = "String", required = true)})
@ -328,77 +332,56 @@ public class CarOpenApi {
int carType = bizContent.getIntValue("carType");
String imgUrl = bizContent.getString("imgUrl");
//去掉 amp;
if(imgUrl!=null){
imgUrl = imgUrl.replace("amp;", "");
}
String dateDir = DateUtils.format(new Date(),DateUtils.DATE_YEAR_MONTH);
String fileName = "car/" + dateDir + "/" +parkCode+"/"+UUIDUtil.uuid() + ".jpg";
if (!StringUtils.isEmpty(imgUrl)) {
URL url = null;
InputStream is = null;
HttpURLConnection httpUrl = null;
try {
if(imgUrl!=null){
imgUrl = imgUrl.replace("amp;", "");
//缓存
CarImageCache carImageCache = new CarImageCache();
carImageCache.setFileName(fileName);
carImageCache.setImgUrl(imgUrl);
redisUtils.set("CarImage:"+fileName,carImageCache,1500l);//保存25分钟60*25
}
//new一个URL对象
url = new URL(imgUrl);
//打开链接
HttpURLConnection conn = (HttpURLConnection)url.openConnection();
//设置请求方式为"GET"
conn.setRequestMethod("GET");
//超时响应时间为5秒
conn.setConnectTimeout(5 * 1000);
//通过输入流获取图片数据
InputStream inStream = conn.getInputStream();
ByteArrayOutputStream outStream = new ByteArrayOutputStream();
//创建一个Buffer字符串
byte[] buffer = new byte[1024000];
//每次读取的字符串长度，如果为-1，代表全部读取完毕
int len = 0;
//使用一个输入流从buffer里把数据读取出来
while( (len=inStream.read(buffer)) != -1 ){
//用输出流往buffer里写入数据，中间参数代表从哪个位置开始读，len代表读取的长度
outStream.write(buffer, 0, len);
}
//得到图片的二进制数据，以二进制封装得到数据，具有通用性
byte[] dataImage = outStream.toByteArray();
//关闭输入流
inStream.close();
outStream.close();
ByteArrayInputStream bais = new ByteArrayInputStream(dataImage);
BufferedImage image = ImageIO.read(bais);
//获取图片的宽、高
// System.out.println("Width: " + image.getWidth());
// System.out.println("Height: " + image.getHeight());
int w = image.getWidth();
int h = image.getHeight();
//调整尺寸不能大于1024
if(w>1024 || h>1024){
image = ThumbnailsUtils.resizeImageOne(image,1024,1024);
}
byte[] data = ImageUtils.imageToBytes(image);
is = new ByteArrayInputStream(data);
int rl = is.available();
PutObjectOptions putObjectOptions = new PutObjectOptions(rl, -1L);
putObjectOptions.setContentType("image/jpeg");
this.minioClient.putObject(this.minioConfig
.getBucketName(), fileName, is, putObjectOptions);
// url = new URL(imgUrl);
// if (!StringUtils.isEmpty(imgUrl)) {
// URL url = null;
// InputStream is = null;
// HttpURLConnection httpUrl = null;
// try {
//
// //new一个URL对象
// url = new URL(imgUrl);
// //打开链接
// HttpURLConnection conn = (HttpURLConnection)url.openConnection();
// //设置请求方式为"GET"
// conn.setRequestMethod("GET");
// //超时响应时间为5秒
// conn.setConnectTimeout(5 * 1000);
// //通过输入流获取图片数据
// InputStream inStream = conn.getInputStream();
//
// ByteArrayOutputStream outStream = new ByteArrayOutputStream();
// //创建一个Buffer字符串
// byte[] buffer = new byte[1024000];
// //每次读取的字符串长度，如果为-1，代表全部读取完毕
// int len = 0;
// //使用一个输入流从buffer里把数据读取出来
// while( (len=inStream.read(buffer)) != -1 ){
// //用输出流往buffer里写入数据，中间参数代表从哪个位置开始读，len代表读取的长度
// outStream.write(buffer, 0, len);
// }
//
// //得到图片的二进制数据，以二进制封装得到数据，具有通用性
// byte[] dataImage = outStream.toByteArray();
//
// //关闭输入流
// inStream.close();
// outStream.close();
//
// ByteArrayInputStream bais = new ByteArrayInputStream(dataImage);
// BufferedImage image = ImageIO.read(bais);
//
// BufferedImage image = ImageIO.read(url);
// //获取图片的宽、高
// System.out.println("Width: " + image.getWidth());
// System.out.println("Height: " + image.getHeight());
// int w = image.getWidth();
// int h = image.getHeight();
// //调整尺寸不能大于1024
@ -412,31 +395,54 @@ public class CarOpenApi {
// putObjectOptions.setContentType("image/jpeg");
// this.minioClient.putObject(this.minioConfig
// .getBucketName(), fileName, is, putObjectOptions);
// httpUrl = (HttpURLConnection) url.openConnection();
// httpUrl.connect();
//// httpUrl.getInputStream();
// is = httpUrl.getInputStream();
// PutObjectOptions putObjectOptions = new PutObjectOptions(is.available(), -1L);
// putObjectOptions.setContentType("image/jpeg");
// this.minioClient.putObject(this.minioConfig
// .getBucketName(), fileName, is, putObjectOptions);
} catch (Exception e) {
e.printStackTrace();
log.error("保存图片失败:"+imgUrl+"--------------------"+e);
fileName = "";
} finally {
if (is != null)
try {
is.close();
} catch (IOException e) {
e.printStackTrace();
}
if (httpUrl != null)
httpUrl.disconnect();
}
}
//
//
//
//
//// url = new URL(imgUrl);
////
//// BufferedImage image = ImageIO.read(url);
//// //获取图片的宽、高
//// System.out.println("Width: " + image.getWidth());
//// System.out.println("Height: " + image.getHeight());
//// int w = image.getWidth();
//// int h = image.getHeight();
//// //调整尺寸不能大于1024
//// if(w>1024 || h>1024){
//// image = ThumbnailsUtils.resizeImageOne(image,1024,1024);
//// }
//// byte[] data = ImageUtils.imageToBytes(image);
//// is = new ByteArrayInputStream(data);
//// int rl = is.available();
//// PutObjectOptions putObjectOptions = new PutObjectOptions(rl, -1L);
//// putObjectOptions.setContentType("image/jpeg");
//// this.minioClient.putObject(this.minioConfig
//// .getBucketName(), fileName, is, putObjectOptions);
//
//
//// httpUrl = (HttpURLConnection) url.openConnection();
//// httpUrl.connect();
////// httpUrl.getInputStream();
//// is = httpUrl.getInputStream();
//// PutObjectOptions putObjectOptions = new PutObjectOptions(is.available(), -1L);
//// putObjectOptions.setContentType("image/jpeg");
//// this.minioClient.putObject(this.minioConfig
//// .getBucketName(), fileName, is, putObjectOptions);
// } catch (Exception e) {
// e.printStackTrace();
// log.error("保存图片失败:"+imgUrl+"--------------------"+e);
// fileName = "";
// } finally {
// if (is != null)
// try {
// is.close();
// } catch (IOException e) {
// e.printStackTrace();
// }
// if (httpUrl != null)
// httpUrl.disconnect();
// }
// }

92
shapelight-admin/src/main/java/net/shapelight/modules/job/task/CarCacheImageTask.java Normal file
View File

@ -0,0 +1,92 @@
package net.shapelight.modules.job.task;
import io.minio.MinioClient;
import io.minio.PutObjectOptions;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import net.shapelight.common.config.MinioConfig;
import net.shapelight.common.utils.ImageUtils;
import net.shapelight.common.utils.RedisUtils;
import net.shapelight.common.utils.StringUtils;
import net.shapelight.common.utils.ThumbnailsUtils;
import net.shapelight.modules.vo.CarImageCache;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
@Component("carCacheImageTask")
@Slf4j
public class CarCacheImageTask implements ITask {
@Autowired
private RedisUtils redisUtils;
@Autowired
private MinioClient minioClient;
@Autowired
private MinioConfig minioConfig;
@Override
public void run(String params){
//缓存
// CarImageCache carImageCache = new CarImageCache();
// carImageCache.setFileName("car/P2/202107/22222.jpg");
// carImageCache.setImgUrl("http://icecloud-prod.oss-cn-shenzhen.aliyuncs.com/P1619325610/image/202107/12/172151_in_1_%E9%99%95A063Y8_38.jpg?Expires=1626083513&OSSAccessKeyId=LTAI4Furp7fmTGDMRhJ95jaX&Signature=0V5rvWbjp%2BEIAHuBegDrRubhGJo%3D");
// redisUtils.set("CarImage:"+"car/P2/202107/100001.jpg",carImageCache,1500l);//保存25分钟60*25
// CarImageCache cacheObject = (CarImageCache)redisUtils.get("CarImage:car/P2/202107/100001.jpg");
CarImageCache cacheObject = (CarImageCache)redisUtils.getPattern("CarImage*");
if (cacheObject!=null) {
String imgUrl = cacheObject.getImgUrl();
String fileName = cacheObject.getFileName();
log.debug("车辆图片地址："+fileName+" url:"+imgUrl);
URL url = null;
InputStream is = null;
HttpURLConnection httpUrl = null;
try {
url = new URL(imgUrl);
BufferedImage image = ImageIO.read(url);
//获取图片的宽、高
// System.out.println("Width: " + image.getWidth());
// System.out.println("Height: " + image.getHeight());
int w = image.getWidth();
int h = image.getHeight();
//调整尺寸不能大于1024
if(w>1024 || h>1024){
image = ThumbnailsUtils.resizeImageOne(image,1024,1024);
}
byte[] data = ImageUtils.imageToBytes(image);
is = new ByteArrayInputStream(data);
int rl = is.available();
PutObjectOptions putObjectOptions = new PutObjectOptions(rl, -1L);
putObjectOptions.setContentType("image/jpeg");
this.minioClient.putObject(this.minioConfig
.getBucketName(), fileName, is, putObjectOptions);
log.debug("保存成功："+this.minioConfig.getBucketName()+fileName);
} catch (Exception e) {
e.printStackTrace();
log.error("保存图片失败:"+imgUrl+":"+e.getMessage());
fileName = "";
} finally {
if (is != null)
try {
is.close();
} catch (IOException e) {
e.printStackTrace();
}
if (httpUrl != null)
httpUrl.disconnect();
}
}
}
}

12
shapelight-admin/src/main/java/net/shapelight/modules/job/task/XaRecordTask.java
View File

@ -1,5 +1,6 @@
package net.shapelight.modules.job.task;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.core.conditions.Wrapper;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
@ -148,6 +149,7 @@ public class XaRecordTask implements ITask {
if(resJson.contains("ErrorLineParameter")){
JSONObject resJsonObject = JSONObject.parseObject(resJson);
String errLine = resJsonObject.getJSONObject("sta").getString("ErrorLineParameter").trim();
JSONArray datas = resJsonObject.getJSONArray("datas");
if(errLine.equals("empty")){
for (TenRecordEntity recordEntity : updateRecords) {
recordEntity.setXaSync(1);
@ -166,6 +168,16 @@ public class XaRecordTask implements ITask {
}
}
}
if(datas!=null){
for(int i = 0;i<datas.size();i++){
JSONObject resData = (JSONObject) datas.get(i);
if(((String)resData.get("Result")).indexOf("请勿重复申报")>0){
TenRecordEntity recordEntity = updateRecords.get(i);
recordEntity.setXaSync(1);
tenRecordService.updateById(recordEntity);
}
}
}
}
}

69
shapelight-admin/src/main/java/net/shapelight/modules/xian/service/XaApi.java
View File

@ -100,33 +100,36 @@ public class XaApi {
HttpURLConnection conn = null;
try {
URL dataurl = new URL(url);
if (dataurl == null) {
return "请求地址为空";
}
conn = (HttpURLConnection) dataurl.openConnection();
try {
conn.setRequestMethod("POST");
String currdate = new SimpleDateFormat("yyyyMMdd").format(new Date());
String token = XaUtils.md5(appid + secre + currdate + json.replaceAll("\r\n", ""));
conn = (HttpURLConnection) dataurl.openConnection();
// try {
conn.setRequestMethod("POST");
String currdate = new SimpleDateFormat("yyyyMMdd").format(new Date());
String token = XaUtils.md5(appid + secre + currdate + json.replaceAll("\r\n", ""));
// System.out.println("json****" + json);
// System.out.println("333appid***" + appid);
// System.out.println("333token***" + token);
// System.out.println("333serviceId***" + serviceId);
// System.out.println("333serviceValue***" + serviceValue);
conn.setRequestProperty("appid", appid);
conn.setRequestProperty("token", token);
conn.setRequestProperty("tranId", tranId);//可固定这么写
conn.setRequestProperty("serviceId", serviceId);
conn.setRequestProperty("serviceValue", serviceValue);
conn.setRequestProperty("versionCode", "");
conn.setRequestProperty("appid", appid);
conn.setRequestProperty("token", token);
conn.setRequestProperty("tranId", tranId);//可固定这么写
conn.setRequestProperty("serviceId", serviceId);
conn.setRequestProperty("serviceValue", serviceValue);
conn.setRequestProperty("versionCode", "");
conn.setDoOutput(true);
conn.setDoInput(true);
conn.setDoOutput(true);
conn.setDoInput(true);
//连接建立超时时间还有读取数据超时时间，
conn.setConnectTimeout(600000);
conn.setReadTimeout(600000);
//连接建立超时时间还有读取数据超时时间，
conn.setConnectTimeout(60000);
conn.setReadTimeout(60000);
conn.connect();
conn.connect();
// String params = "appid=" + URLEncoder.encode(appid, "UTF-8") + "&appsecret=" + URLEncoder.encode(secre, "UTF-8");
// OutputStream outputStream = conn.getOutputStream();
@ -134,16 +137,16 @@ public class XaApi {
// outputStream.flush();
// outputStream.close();
OutputStreamWriter os = new OutputStreamWriter(conn.getOutputStream(), "UTF-8");
os.write(json);
os.flush();
os.close();
//conn.setConnectTimeout(10000);
} catch (Exception ex) {
System.out.println("errir=" + ex.getMessage());
conn.disconnect();
return "";
}
OutputStreamWriter os = new OutputStreamWriter(conn.getOutputStream(), "UTF-8");
os.write(json);
os.flush();
os.close();
//conn.setConnectTimeout(10000);
// } catch (Exception ex) {
// System.out.println("errir=" + ex.getMessage());
// conn.disconnect();
// return ex.getMessage();
// }
InputStream is = conn.getInputStream();
DataInputStream dis = new DataInputStream(is);
byte bt[] = new byte[dis.available()];
@ -153,10 +156,10 @@ public class XaApi {
is.close();
dis.close();
} catch (Exception e) {
log.error("请求失败"+url+"\n"+e.getMessage());
data = "";
log.error("请求失败" + url + "\n" + e.getMessage());
data = e.getMessage();
} finally {
if(conn!=null){
if (conn != null) {
conn.disconnect();
}
}
@ -442,7 +445,7 @@ public class XaApi {
BufferedReader br = new BufferedReader(isr);
String lineTxt = null;
while ((lineTxt = br.readLine()) != null) {
stringBuffer.append(lineTxt.replace(" ",""));
stringBuffer.append(lineTxt.replace(" ", ""));
}
br.close();
} else {
@ -458,7 +461,7 @@ public class XaApi {
JSONObject xml = JSONObject.parseObject(stringBuffer.toString());
JSONObject data = xml.getJSONObject("xml").getJSONObject("data");
JSONArray hlwtable = data.getJSONArray("HLWTable");
List<TenAddressEntity> list = new ArrayList<>();
List<TenAddressEntity> list = new ArrayList<>();
for (int i = 0; i < hlwtable.size(); i++) {
JSONObject object = (JSONObject) hlwtable.get(i);
@ -485,7 +488,7 @@ public class XaApi {
addressEntity.setDzdm(object.getString("@DZDM"));
addressEntity.setJlxdm(object.getString("@JLXDM"));
list.add(addressEntity);
list.add(addressEntity);
// tenAddressService.saveOrUpdate(addressEntity);
}

2
shapelight-admin/src/main/resources/application-dev.yml
View File

@ -3,7 +3,7 @@ spring:
type: com.alibaba.druid.pool.DruidDataSource
druid:
driver-class-name: com.mysql.cj.jdbc.Driver
url: jdbc:mysql://192.168.50.232:3306/cell_db_0621?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&useSSL=false&rewriteBatchedStatements=true
url: jdbc:mysql://192.168.50.20:3306/cell_db_0621?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&useSSL=false&rewriteBatchedStatements=true
username: user
password: user@server001
initial-size: 10

10
shapelight-admin/src/main/resources/application.yml
View File

@ -32,7 +32,7 @@ spring:
file-size-threshold: 1000MB
redis:
database: 0
host: 192.168.50.232
host: 192.168.50.20
port: 6379
password: 123456 # 密码（默认为空）
timeout: 6000ms # 连接超时时长（毫秒）
@ -84,11 +84,17 @@ global:
db_host: localhost
db_port: 3306
minio:
endpoint: http://192.168.50.232:9000
endpoint: http://192.168.50.20:9000
port: 9000
accessKey: admin
secretKey: admin@C501
bucketName: cell
# minio:
# endpoint: https://c.xitu3d.com:19000
# port: 19000
# accessKey: 7885444af42e4b30c518c5be17c8850b
# secretKey: 0fad3477fb9f60c7be75561db967e8d7
# bucketName: cell
device:
appId: 8e9b941923abd
appSecret: Mjk0ODIxOTcwMTQ0NjkwNzA1NDE0MzcyNjgxMTQ0MDc0MjE

34
shapelight-admin/src/test/java/gb/CarImageTest.java
View File

@ -1,15 +1,14 @@
package gb;
import com.alibaba.fastjson.JSONObject;
import io.minio.MinioClient;
import io.minio.PutObjectOptions;
import lombok.extern.slf4j.Slf4j;
import net.coobird.thumbnailator.Thumbnails;
import net.shapelight.AdminApplication;
import net.shapelight.common.config.MinioConfig;
import net.shapelight.common.utils.ImageUtils;
import net.shapelight.common.utils.StringUtils;
import net.shapelight.common.utils.ThumbnailsUtils;
import net.shapelight.common.utils.UUIDUtil;
import net.shapelight.common.utils.*;
import net.shapelight.modules.vo.CarImageCache;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
@ -22,6 +21,8 @@ import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Date;
import java.util.Map;
import java.util.Random;
@RunWith(SpringRunner.class)
@ -32,15 +33,34 @@ public class CarImageTest {
private MinioClient minioClient;
@Autowired
private MinioConfig minioConfig;
@Autowired
private RedisUtils redisUtils;
@Test
public void ossTest() {
String imgUrl = "http://icecloud-prod.oss-cn-shenzhen.aliyuncs.com/P1613901148/image/202106/17/%E9%99%95A3U32K_out_2_4472.jpg?Expires=1623895438&OSSAccessKeyId=LTAI4Furp7fmTGDMRhJ95jaX&Signature=iJ7H8%2FmOQz2Zs81APGk2SMrREGQ%3D";
imgUrl = imgUrl.replace("amp;", "");
String fileName = "car/" + 222 + "/" + UUIDUtil.uuid() + ".jpg";
// String imgUrl = "http://icecloud-prod.oss-cn-shenzhen.aliyuncs.com/P1613901148/image/202106/17/%E9%99%95A3U32K_out_2_4472.jpg?Expires=1623895438&OSSAccessKeyId=LTAI4Furp7fmTGDMRhJ95jaX&Signature=iJ7H8%2FmOQz2Zs81APGk2SMrREGQ%3D";
// imgUrl = imgUrl.replace("amp;", "");
// String fileName = "car/" + 222 + "/" + UUIDUtil.uuid() + ".jpg";
// //缓存
// JSONObject cache = new JSONObject();
// cache.put("fileName",fileName);
// cache.put("imgUrl",imgUrl);
// redisUtils.set("CarImage:"+fileName,cache,1500l);//保存25分钟60*25
//缓存
CarImageCache carImageCache = new CarImageCache();
carImageCache.setFileName("car/P2/202107/22222.jpg");
carImageCache.setImgUrl("http://icecloud-prod.oss-cn-shenzhen.aliyuncs.com/P1619325610/image/202107/12/172151_in_1_%E9%99%95A063Y8_38.jpg?Expires=1626083513&OSSAccessKeyId=LTAI4Furp7fmTGDMRhJ95jaX&Signature=0V5rvWbjp%2BEIAHuBegDrRubhGJo%3D");
redisUtils.set("CarImage:"+"car/P2/202107/"+ UUIDUtil.uuid()+".jpg",carImageCache,1500l);//保存25分钟60*25
// JSONObject cacheObject = redisUtils.getPattern("CarImage");
// CarImageCache cacheObject = (CarImageCache)redisUtils.get("CarImage:car/P2/202107/100001.jpg");
CarImageCache cacheObject = (CarImageCache)redisUtils.getPattern("CarImage*");
String imgUrl = cacheObject.getImgUrl();
String fileName = cacheObject.getFileName();
if (!StringUtils.isEmpty(imgUrl)) {
URL url = null;
InputStream is = null;

36
shapelight-common/src/main/java/net/shapelight/common/utils/RedisUtils.java
View File

@ -3,13 +3,17 @@
package net.shapelight.common.utils;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.google.gson.JsonObject;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.*;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.io.Serializable;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
@ -320,4 +324,36 @@ public class RedisUtils {
redisTemplate.expire(key, 5, TimeUnit.SECONDS);
}
}
/**
* 获取第一个
*
* @param pattern
*/
public Object getPattern(final String pattern) {
Set<Serializable> keys = redisTemplate.keys(pattern);
Serializable minKey = null;
long ex = 1500;
if (keys.size() > 0){
for(Serializable key: keys){
// Object v = get((String)key);
long t = redisTemplate.opsForValue().getOperations().getExpire(key);
if(t<ex){
ex = t;
minKey = key;
}
// if(t>1400){
// continue;
// }
// remove((String)key);
// return v;
}
if(ex < 1400){
Object v = get((String)minKey);
remove((String)minKey);
return v;
}
}
return null;
}
}