v2.0.15

1.优化车辆图片并发高，用redis缓存，定时器消费处理

shapelight-admin/src/main/java/net/shapelight/modules/car/controller/CarOpenApi.java

@@ -25,6 +25,7 @@ import net.shapelight.common.config.MinioConfig;
 import net.shapelight.common.utils.*;
 import net.shapelight.modules.ten.entity.*;
 import net.shapelight.modules.ten.service.*;
+import net.shapelight.modules.vo.CarImageCache;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -70,6 +71,9 @@ public class CarOpenApi {
     @Autowired
     private TenPackChannalService tenPackChannalService;
 
+    @Autowired
+    private RedisUtils redisUtils;
+
     @PostMapping({"/getMsg/{cellId}"})
     @ApiOperation("获取MSG")
     @ApiImplicitParams({@ApiImplicitParam(name = "sn", value = "设备sn", paramType = "query", dataType = "String", required = true)})
@@ -328,77 +332,56 @@ public class CarOpenApi {
         int carType = bizContent.getIntValue("carType");
         String imgUrl = bizContent.getString("imgUrl");
         //去掉 amp;
-        if(imgUrl!=null){
-            imgUrl = imgUrl.replace("amp;", "");
-        }
-
-
         String dateDir = DateUtils.format(new Date(),DateUtils.DATE_YEAR_MONTH);
         String fileName = "car/" + dateDir + "/" +parkCode+"/"+UUIDUtil.uuid() + ".jpg";
-        if (!StringUtils.isEmpty(imgUrl)) {
+        if(imgUrl!=null){
-            URL url = null;
+            imgUrl = imgUrl.replace("amp;", "");
-            InputStream is = null;
+            //缓存
-            HttpURLConnection httpUrl = null;
+            CarImageCache carImageCache = new CarImageCache();
-            try {
+            carImageCache.setFileName(fileName);
-
+            carImageCache.setImgUrl(imgUrl);
-                //new一个URL对象
+            redisUtils.set("CarImage:"+fileName,carImageCache,1500l);//保存25分钟60*25
-                url = new URL(imgUrl);
-                //打开链接
-                HttpURLConnection conn = (HttpURLConnection)url.openConnection();
-                //设置请求方式为"GET"
-                conn.setRequestMethod("GET");
-                //超时响应时间为5秒
-                conn.setConnectTimeout(5 * 1000);
-                //通过输入流获取图片数据
-                InputStream inStream = conn.getInputStream();
-
-                ByteArrayOutputStream outStream = new ByteArrayOutputStream();
-                //创建一个Buffer字符串
-                byte[] buffer = new byte[1024000];
-                //每次读取的字符串长度，如果为-1，代表全部读取完毕
-                int len = 0;
-                //使用一个输入流从buffer里把数据读取出来
-                while( (len=inStream.read(buffer)) != -1 ){
-                    //用输出流往buffer里写入数据，中间参数代表从哪个位置开始读，len代表读取的长度
-                    outStream.write(buffer, 0, len);
         }
 
-                //得到图片的二进制数据，以二进制封装得到数据，具有通用性
+//        if (!StringUtils.isEmpty(imgUrl)) {
-                byte[] dataImage = outStream.toByteArray();
+//            URL url = null;
-
+//            InputStream is = null;
-                //关闭输入流
+//            HttpURLConnection httpUrl = null;
-                inStream.close();
+//            try {
-                outStream.close();
+//
-
+//                //new一个URL对象
-                ByteArrayInputStream bais = new ByteArrayInputStream(dataImage);
+//                url = new URL(imgUrl);
-                BufferedImage image = ImageIO.read(bais);
+//                //打开链接
-
+//                HttpURLConnection conn = (HttpURLConnection)url.openConnection();
-                //获取图片的宽、高
+//                //设置请求方式为"GET"
-//                System.out.println("Width: " + image.getWidth());
+//                conn.setRequestMethod("GET");
-//                System.out.println("Height: " + image.getHeight());
+//                //超时响应时间为5秒
-                int w = image.getWidth();
+//                conn.setConnectTimeout(5 * 1000);
-                int h = image.getHeight();
+//                //通过输入流获取图片数据
-                //调整尺寸不能大于1024
+//                InputStream inStream = conn.getInputStream();
-                if(w>1024 || h>1024){
+//
-                    image = ThumbnailsUtils.resizeImageOne(image,1024,1024);
+//                ByteArrayOutputStream outStream = new ByteArrayOutputStream();
-                }
+//                //创建一个Buffer字符串
-                byte[] data = ImageUtils.imageToBytes(image);
+//                byte[] buffer = new byte[1024000];
-                is = new ByteArrayInputStream(data);
+//                //每次读取的字符串长度，如果为-1，代表全部读取完毕
-                int rl = is.available();
+//                int len = 0;
-                PutObjectOptions putObjectOptions = new PutObjectOptions(rl, -1L);
+//                //使用一个输入流从buffer里把数据读取出来
-                putObjectOptions.setContentType("image/jpeg");
+//                while( (len=inStream.read(buffer)) != -1 ){
-                this.minioClient.putObject(this.minioConfig
+//                    //用输出流往buffer里写入数据，中间参数代表从哪个位置开始读，len代表读取的长度
-                        .getBucketName(), fileName, is, putObjectOptions);
+//                    outStream.write(buffer, 0, len);
-
+//                }
-
+//
-
+//                //得到图片的二进制数据，以二进制封装得到数据，具有通用性
-
+//                byte[] dataImage = outStream.toByteArray();
-//                url = new URL(imgUrl);
+//
+//                //关闭输入流
+//                inStream.close();
+//                outStream.close();
+//
+//                ByteArrayInputStream bais = new ByteArrayInputStream(dataImage);
+//                BufferedImage image = ImageIO.read(bais);
 //
-//                BufferedImage image = ImageIO.read(url);
 //                //获取图片的宽、高
-//                System.out.println("Width: " + image.getWidth());
-//                System.out.println("Height: " + image.getHeight());
 //                int w = image.getWidth();
 //                int h = image.getHeight();
 //                //调整尺寸不能大于1024
@@ -412,31 +395,54 @@ public class CarOpenApi {
 //                putObjectOptions.setContentType("image/jpeg");
 //                this.minioClient.putObject(this.minioConfig
 //                        .getBucketName(), fileName, is, putObjectOptions);
-
+//
-
+//
-//                httpUrl = (HttpURLConnection) url.openConnection();
+//
-//                httpUrl.connect();
+//
-////                httpUrl.getInputStream();
+////                url = new URL(imgUrl);
-//                is = httpUrl.getInputStream();
+////
-//                PutObjectOptions putObjectOptions = new PutObjectOptions(is.available(), -1L);
+////                BufferedImage image = ImageIO.read(url);
-//                putObjectOptions.setContentType("image/jpeg");
+////                //获取图片的宽、高
-//                this.minioClient.putObject(this.minioConfig
+////                System.out.println("Width: " + image.getWidth());
-//                        .getBucketName(), fileName, is, putObjectOptions);
+////                System.out.println("Height: " + image.getHeight());
-            } catch (Exception e) {
+////                int w = image.getWidth();
-                e.printStackTrace();
+////                int h = image.getHeight();
-                log.error("保存图片失败:"+imgUrl+"--------------------"+e);
+////                //调整尺寸不能大于1024
-                fileName = "";
+////                if(w>1024 || h>1024){
-            } finally {
+////                    image = ThumbnailsUtils.resizeImageOne(image,1024,1024);
-                if (is != null)
+////                }
-                    try {
+////                byte[] data = ImageUtils.imageToBytes(image);
-                        is.close();
+////                is = new ByteArrayInputStream(data);
-                    } catch (IOException e) {
+////                int rl = is.available();
-                        e.printStackTrace();
+////                PutObjectOptions putObjectOptions = new PutObjectOptions(rl, -1L);
-                    }
+////                putObjectOptions.setContentType("image/jpeg");
-                if (httpUrl != null)
+////                this.minioClient.putObject(this.minioConfig
-                    httpUrl.disconnect();
+////                        .getBucketName(), fileName, is, putObjectOptions);
-            }
+//
-        }
+//
+////                httpUrl = (HttpURLConnection) url.openConnection();
+////                httpUrl.connect();
+//////                httpUrl.getInputStream();
+////                is = httpUrl.getInputStream();
+////                PutObjectOptions putObjectOptions = new PutObjectOptions(is.available(), -1L);
+////                putObjectOptions.setContentType("image/jpeg");
+////                this.minioClient.putObject(this.minioConfig
+////                        .getBucketName(), fileName, is, putObjectOptions);
+//            } catch (Exception e) {
+//                e.printStackTrace();
+//                log.error("保存图片失败:"+imgUrl+"--------------------"+e);
+//                fileName = "";
+//            } finally {
+//                if (is != null)
+//                    try {
+//                        is.close();
+//                    } catch (IOException e) {
+//                        e.printStackTrace();
+//                    }
+//                if (httpUrl != null)
+//                    httpUrl.disconnect();
+//            }
+//        }
 
 
 

shapelight-admin/src/main/java/net/shapelight/modules/job/task/CarCacheImageTask.java

@@ -0,0 +1,92 @@
+package net.shapelight.modules.job.task;
+
+
+import io.minio.MinioClient;
+import io.minio.PutObjectOptions;
+import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
+import net.shapelight.common.config.MinioConfig;
+import net.shapelight.common.utils.ImageUtils;
+import net.shapelight.common.utils.RedisUtils;
+import net.shapelight.common.utils.StringUtils;
+import net.shapelight.common.utils.ThumbnailsUtils;
+import net.shapelight.modules.vo.CarImageCache;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import javax.imageio.ImageIO;
+import java.awt.image.BufferedImage;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
+
+@Component("carCacheImageTask")
+@Slf4j
+public class CarCacheImageTask implements ITask {
+
+    @Autowired
+    private RedisUtils redisUtils;
+    @Autowired
+    private MinioClient minioClient;
+    @Autowired
+    private MinioConfig minioConfig;
+
+    @Override
+    public void run(String params){
+        //缓存
+//        CarImageCache carImageCache = new CarImageCache();
+//        carImageCache.setFileName("car/P2/202107/22222.jpg");
+//        carImageCache.setImgUrl("http://icecloud-prod.oss-cn-shenzhen.aliyuncs.com/P1619325610/image/202107/12/172151_in_1_%E9%99%95A063Y8_38.jpg?Expires=1626083513&OSSAccessKeyId=LTAI4Furp7fmTGDMRhJ95jaX&Signature=0V5rvWbjp%2BEIAHuBegDrRubhGJo%3D");
+//        redisUtils.set("CarImage:"+"car/P2/202107/100001.jpg",carImageCache,1500l);//保存25分钟60*25
+
+
+//        CarImageCache cacheObject = (CarImageCache)redisUtils.get("CarImage:car/P2/202107/100001.jpg");
+        CarImageCache cacheObject = (CarImageCache)redisUtils.getPattern("CarImage*");
+
+        if (cacheObject!=null) {
+            String imgUrl = cacheObject.getImgUrl();
+            String fileName = cacheObject.getFileName();
+            log.debug("车辆图片地址："+fileName+"   url:"+imgUrl);
+            URL url = null;
+            InputStream is = null;
+            HttpURLConnection httpUrl = null;
+            try {
+                url = new URL(imgUrl);
+                BufferedImage image = ImageIO.read(url);
+                //获取图片的宽、高
+//                System.out.println("Width: " + image.getWidth());
+//                System.out.println("Height: " + image.getHeight());
+                int w = image.getWidth();
+                int h = image.getHeight();
+                //调整尺寸不能大于1024
+                if(w>1024 || h>1024){
+                    image = ThumbnailsUtils.resizeImageOne(image,1024,1024);
+                }
+                byte[] data = ImageUtils.imageToBytes(image);
+                is = new ByteArrayInputStream(data);
+                int rl = is.available();
+                PutObjectOptions putObjectOptions = new PutObjectOptions(rl, -1L);
+                putObjectOptions.setContentType("image/jpeg");
+                this.minioClient.putObject(this.minioConfig
+                        .getBucketName(), fileName, is, putObjectOptions);
+                log.debug("保存成功："+this.minioConfig.getBucketName()+fileName);
+            } catch (Exception e) {
+                e.printStackTrace();
+                log.error("保存图片失败:"+imgUrl+":"+e.getMessage());
+                fileName = "";
+            } finally {
+                if (is != null)
+                    try {
+                        is.close();
+                    } catch (IOException e) {
+                        e.printStackTrace();
+                    }
+                if (httpUrl != null)
+                    httpUrl.disconnect();
+            }
+        }
+
+    }
+}

shapelight-admin/src/main/java/net/shapelight/modules/job/task/XaRecordTask.java

@@ -1,5 +1,6 @@
 package net.shapelight.modules.job.task;
 
+import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.core.conditions.Wrapper;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
@@ -148,6 +149,7 @@ public class XaRecordTask implements ITask {
         if(resJson.contains("ErrorLineParameter")){
             JSONObject resJsonObject = JSONObject.parseObject(resJson);
             String errLine = resJsonObject.getJSONObject("sta").getString("ErrorLineParameter").trim();
+            JSONArray datas = resJsonObject.getJSONArray("datas");
             if(errLine.equals("empty")){
                 for (TenRecordEntity recordEntity : updateRecords) {
                     recordEntity.setXaSync(1);
@@ -166,6 +168,16 @@ public class XaRecordTask implements ITask {
                     }
                 }
             }
+            if(datas!=null){
+                for(int i = 0;i<datas.size();i++){
+                    JSONObject resData = (JSONObject) datas.get(i);
+                    if(((String)resData.get("Result")).indexOf("请勿重复申报")>0){
+                        TenRecordEntity recordEntity = updateRecords.get(i);
+                        recordEntity.setXaSync(1);
+                        tenRecordService.updateById(recordEntity);
+                    }
+                }
+            }
         }
     }
 

shapelight-admin/src/main/java/net/shapelight/modules/xian/service/XaApi.java

@@ -100,9 +100,12 @@ public class XaApi {
         HttpURLConnection conn = null;
         try {
             URL dataurl = new URL(url);
+            if (dataurl == null) {
+                return "请求地址为空";
+            }
 
             conn = (HttpURLConnection) dataurl.openConnection();
-            try {
+//            try {
             conn.setRequestMethod("POST");
             String currdate = new SimpleDateFormat("yyyyMMdd").format(new Date());
             String token = XaUtils.md5(appid + secre + currdate + json.replaceAll("\r\n", ""));
@@ -123,8 +126,8 @@ public class XaApi {
             conn.setDoInput(true);
 
             //连接建立超时时间还有读取数据超时时间，
-                conn.setConnectTimeout(600000);
+            conn.setConnectTimeout(60000);
-                conn.setReadTimeout(600000);
+            conn.setReadTimeout(60000);
 
             conn.connect();
 
@@ -139,11 +142,11 @@ public class XaApi {
             os.flush();
             os.close();
             //conn.setConnectTimeout(10000);
-            } catch (Exception ex) {
+//            } catch (Exception ex) {
-                System.out.println("errir=" + ex.getMessage());
+//                System.out.println("errir=" + ex.getMessage());
-                conn.disconnect();
+//                conn.disconnect();
-                return "";
+//                return ex.getMessage();
-            }
+//            }
             InputStream is = conn.getInputStream();
             DataInputStream dis = new DataInputStream(is);
             byte bt[] = new byte[dis.available()];
@@ -154,7 +157,7 @@ public class XaApi {
             dis.close();
         } catch (Exception e) {
             log.error("请求失败" + url + "\n" + e.getMessage());
-            data = "";
+            data = e.getMessage();
         } finally {
             if (conn != null) {
                 conn.disconnect();

shapelight-admin/src/main/resources/application-dev.yml

@@ -3,7 +3,7 @@ spring:
     type: com.alibaba.druid.pool.DruidDataSource
     druid:
       driver-class-name: com.mysql.cj.jdbc.Driver
-      url: jdbc:mysql://192.168.50.232:3306/cell_db_0621?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&useSSL=false&rewriteBatchedStatements=true
+      url: jdbc:mysql://192.168.50.20:3306/cell_db_0621?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&useSSL=false&rewriteBatchedStatements=true
       username: user
       password: user@server001
       initial-size: 10

shapelight-admin/src/main/resources/application.yml

@@ -32,7 +32,7 @@ spring:
       file-size-threshold: 1000MB
   redis:
     database: 0
-    host: 192.168.50.232
+    host: 192.168.50.20
     port: 6379
     password: 123456      # 密码（默认为空）
     timeout: 6000ms  # 连接超时时长（毫秒）
@@ -84,11 +84,17 @@ global:
     db_host: localhost
     db_port: 3306
   minio:
-    endpoint: http://192.168.50.232:9000
+    endpoint: http://192.168.50.20:9000
     port: 9000
     accessKey: admin
     secretKey: admin@C501
     bucketName: cell
+#  minio:
+#    endpoint: https://c.xitu3d.com:19000
+#    port: 19000
+#    accessKey: 7885444af42e4b30c518c5be17c8850b
+#    secretKey: 0fad3477fb9f60c7be75561db967e8d7
+#    bucketName: cell
   device:
     appId: 8e9b941923abd
     appSecret: Mjk0ODIxOTcwMTQ0NjkwNzA1NDE0MzcyNjgxMTQ0MDc0MjE

shapelight-admin/src/test/java/gb/CarImageTest.java

@@ -1,15 +1,14 @@
 package gb;
 
+import com.alibaba.fastjson.JSONObject;
 import io.minio.MinioClient;
 import io.minio.PutObjectOptions;
 import lombok.extern.slf4j.Slf4j;
 import net.coobird.thumbnailator.Thumbnails;
 import net.shapelight.AdminApplication;
 import net.shapelight.common.config.MinioConfig;
-import net.shapelight.common.utils.ImageUtils;
+import net.shapelight.common.utils.*;
-import net.shapelight.common.utils.StringUtils;
+import net.shapelight.modules.vo.CarImageCache;
-import net.shapelight.common.utils.ThumbnailsUtils;
-import net.shapelight.common.utils.UUIDUtil;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -22,6 +21,8 @@ import java.io.*;
 import java.net.HttpURLConnection;
 import java.net.URL;
 import java.util.Date;
+import java.util.Map;
+import java.util.Random;
 
 
 @RunWith(SpringRunner.class)
@@ -32,15 +33,34 @@ public class CarImageTest {
     private MinioClient minioClient;
     @Autowired
     private MinioConfig minioConfig;
+    @Autowired
+    private RedisUtils redisUtils;
 
     @Test
     public void ossTest() {
 
-        String imgUrl = "http://icecloud-prod.oss-cn-shenzhen.aliyuncs.com/P1613901148/image/202106/17/%E9%99%95A3U32K_out_2_4472.jpg?Expires=1623895438&OSSAccessKeyId=LTAI4Furp7fmTGDMRhJ95jaX&Signature=iJ7H8%2FmOQz2Zs81APGk2SMrREGQ%3D";
+//        String imgUrl = "http://icecloud-prod.oss-cn-shenzhen.aliyuncs.com/P1613901148/image/202106/17/%E9%99%95A3U32K_out_2_4472.jpg?Expires=1623895438&OSSAccessKeyId=LTAI4Furp7fmTGDMRhJ95jaX&Signature=iJ7H8%2FmOQz2Zs81APGk2SMrREGQ%3D";
-        imgUrl = imgUrl.replace("amp;", "");
+//        imgUrl = imgUrl.replace("amp;", "");
-        String fileName = "car/" + 222 + "/" + UUIDUtil.uuid() + ".jpg";
+//        String fileName = "car/" + 222 + "/" + UUIDUtil.uuid() + ".jpg";
+//        //缓存
+//            JSONObject cache = new JSONObject();
+//            cache.put("fileName",fileName);
+//            cache.put("imgUrl",imgUrl);
+//            redisUtils.set("CarImage:"+fileName,cache,1500l);//保存25分钟60*25
 
 
+
+        //缓存
+        CarImageCache carImageCache = new CarImageCache();
+        carImageCache.setFileName("car/P2/202107/22222.jpg");
+        carImageCache.setImgUrl("http://icecloud-prod.oss-cn-shenzhen.aliyuncs.com/P1619325610/image/202107/12/172151_in_1_%E9%99%95A063Y8_38.jpg?Expires=1626083513&OSSAccessKeyId=LTAI4Furp7fmTGDMRhJ95jaX&Signature=0V5rvWbjp%2BEIAHuBegDrRubhGJo%3D");
+        redisUtils.set("CarImage:"+"car/P2/202107/"+ UUIDUtil.uuid()+".jpg",carImageCache,1500l);//保存25分钟60*25
+
+//        JSONObject cacheObject = redisUtils.getPattern("CarImage");
+//        CarImageCache cacheObject = (CarImageCache)redisUtils.get("CarImage:car/P2/202107/100001.jpg");
+        CarImageCache cacheObject = (CarImageCache)redisUtils.getPattern("CarImage*");
+        String imgUrl = cacheObject.getImgUrl();
+        String fileName = cacheObject.getFileName();
         if (!StringUtils.isEmpty(imgUrl)) {
             URL url = null;
             InputStream is = null;

shapelight-common/src/main/java/net/shapelight/common/utils/RedisUtils.java

@@ -3,13 +3,17 @@
 package net.shapelight.common.utils;
 
 import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONObject;
+import com.google.gson.JsonObject;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.redis.core.*;
 import org.springframework.stereotype.Component;
 
 import javax.annotation.Resource;
 import java.io.Serializable;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
@@ -320,4 +324,36 @@ public class RedisUtils {
             redisTemplate.expire(key, 5, TimeUnit.SECONDS);
         }
     }
+
+    /**
+     * 获取第一个
+     *
+     * @param pattern
+     */
+    public Object getPattern(final String pattern) {
+        Set<Serializable> keys = redisTemplate.keys(pattern);
+        Serializable minKey = null;
+        long ex = 1500;
+        if (keys.size() > 0){
+            for(Serializable key: keys){
+//                Object v = get((String)key);
+                long t = redisTemplate.opsForValue().getOperations().getExpire(key);
+                if(t<ex){
+                    ex = t;
+                    minKey = key;
+                }
+//                if(t>1400){
+//                    continue;
+//                }
+//                remove((String)key);
+//                return  v;
+            }
+            if(ex < 1400){
+                Object v = get((String)minKey);
+                remove((String)minKey);
+                return v;
+            }
+        }
+        return null;
+    }
 }