DJL配合javacv实现视频车辆检测

1、pom文件

<?xml version="1.0" encoding="UTF-8"?> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <groupId>org.example</groupId> <artifactId>learndjl</artifactId> <version>1.0</version> <properties> <maven.compiler.source>11</maven.compiler.source> <maven.compiler.target>11</maven.compiler.target> <djl.version>0.35.1</djl.version> <javacpp.platform.macosx-x86_64>macosx-x86_64</javacpp.platform.macosx-x86_64> <javacpp.platform.linux-x86_64>linux-x86_64</javacpp.platform.linux-x86_64> <javacpp.platform.windows-x86_64>windows-x86_64</javacpp.platform.windows-x86_64> <javacv.version>1.5.11</javacv.version> <javacv.ffmpeg.version>7.1-1.5.11</javacv.ffmpeg.version> <javacv.opencv.version>4.10.0-1.5.11</javacv.opencv.version> </properties> <dependencyManagement> <dependencies> <dependency> <groupId>ai.djl</groupId> <artifactId>bom</artifactId> <version>${djl.version}</version> <type>pom</type> <scope>import</scope> </dependency> </dependencies> </dependencyManagement> <dependencies> <dependency> <groupId>commons-cli</groupId> <artifactId>commons-cli</artifactId> <version>1.9.0</version> </dependency> <dependency> <groupId>commons-io</groupId> <artifactId>commons-io</artifactId> <version>2.17.0</version> </dependency> <dependency> <groupId>org.apache.logging.log4j</groupId> <artifactId>log4j-slf4j2-impl</artifactId> <version>2.24.1</version> </dependency> <dependency> <groupId>ai.djl</groupId> <artifactId>api</artifactId> </dependency> <dependency> <groupId>ai.djl</groupId> <artifactId>basicdataset</artifactId> </dependency> <dependency> <groupId>ai.djl</groupId> <artifactId>model-zoo</artifactId> </dependency> <dependency> <groupId>ai.djl.timeseries</groupId> <artifactId>timeseries</artifactId> </dependency> <dependency> <groupId>ai.djl.huggingface</groupId> <artifactId>tokenizers</artifactId> </dependency> <dependency> <groupId>ai.djl.audio</groupId> <artifactId>audio</artifactId> </dependency> <!-- MXNet --> <dependency> <groupId>ai.djl.mxnet</groupId> <artifactId>mxnet-model-zoo</artifactId> <scope>runtime</scope> </dependency> <!-- Pytorch --> <dependency> <groupId>ai.djl.pytorch</groupId> <artifactId>pytorch-model-zoo</artifactId> <scope>runtime</scope> </dependency> <!-- TensorFlow --> <dependency> <groupId>ai.djl.tensorflow</groupId> <artifactId>tensorflow-model-zoo</artifactId> <scope>runtime</scope> </dependency> <!-- ONNXRuntime --> <dependency> <groupId>ai.djl.onnxruntime</groupId> <artifactId>onnxruntime-engine</artifactId> <scope>runtime</scope> <exclusions> <exclusion> <groupId>com.microsoft.onnxruntime</groupId> <artifactId>onnxruntime</artifactId> </exclusion> </exclusions> </dependency> <dependency> <groupId>com.microsoft.onnxruntime</groupId> <artifactId>onnxruntime</artifactId> <version>1.20.0</version> </dependency> <dependency> <groupId>org.testng</groupId> <artifactId>testng</artifactId> <version>7.10.2</version> <scope>test</scope> </dependency> <!-- https://mvnrepository.com/artifact/org.bytedeco/javacv --> <dependency> <groupId>org.bytedeco</groupId> <artifactId>javacv</artifactId> <version>${javacv.version}</version> </dependency> <!--javacv 精简依赖 只依赖windows64位ffmpeg--> <dependency> <groupId>org.bytedeco</groupId> <artifactId>javacpp</artifactId> <version>${javacv.version}</version> <classifier>${javacpp.platform.windows-x86_64}</classifier> </dependency> <dependency> <groupId>org.bytedeco</groupId> <artifactId>ffmpeg</artifactId> <version>${javacv.ffmpeg.version}</version> <classifier>${javacpp.platform.windows-x86_64}</classifier> </dependency> <!-- https://mvnrepository.com/artifact/org.bytedeco/opencv-platform --> <dependency> <groupId>org.bytedeco</groupId> <artifactId>opencv</artifactId> <version>4.10.0-1.5.11</version> <classifier>${javacpp.platform.windows-x86_64}</classifier> </dependency> <!-- https://mvnrepository.com/artifact/org.bytedeco/tesseract --> <dependency> <groupId>org.bytedeco</groupId> <artifactId>tesseract</artifactId> <version>5.5.0-1.5.11</version> <classifier>${javacpp.platform.windows-x86_64}</classifier> </dependency> <!-- https://mvnrepository.com/artifact/org.bytedeco/openblas --> <dependency> <groupId>org.bytedeco</groupId> <artifactId>openblas</artifactId> <version>0.3.28-1.5.11</version> <classifier>${javacpp.platform.windows-x86_64}</classifier> </dependency> <!-- https://mvnrepository.com/artifact/org.bytedeco/leptonica --> <dependency> <groupId>org.bytedeco</groupId> <artifactId>leptonica</artifactId> <version>1.85.0-1.5.11</version> <classifier>${javacpp.platform.windows-x86_64}</classifier> </dependency> <!-- https://mvnrepository.com/artifact/org.bytedeco/flycapture --> <dependency> <groupId>org.bytedeco</groupId> <artifactId>flycapture</artifactId> <version>2.13.3.31-1.5.9</version> <classifier>${javacpp.platform.windows-x86_64}</classifier> </dependency> <!-- https://mvnrepository.com/artifact/org.bytedeco/libdc1394 --> <dependency> <groupId>org.bytedeco</groupId> <artifactId>libdc1394</artifactId> <version>2.2.6-1.5.9</version> <classifier>${javacpp.platform.windows-x86_64}</classifier> </dependency> <!-- https://mvnrepository.com/artifact/org.bytedeco/libfreenect --> <dependency> <groupId>org.bytedeco</groupId> <artifactId>libfreenect</artifactId> <version>0.5.7-1.5.9</version> <classifier>${javacpp.platform.windows-x86_64}</classifier> </dependency> <!-- https://mvnrepository.com/artifact/org.bytedeco/libfreenect2 --> <dependency> <groupId>org.bytedeco</groupId> <artifactId>libfreenect2</artifactId> <version>0.2.0-1.5.9</version> <classifier>${javacpp.platform.windows-x86_64}</classifier> </dependency> <!-- https://mvnrepository.com/artifact/org.bytedeco/librealsense --> <dependency> <groupId>org.bytedeco</groupId> <artifactId>librealsense</artifactId> <version>1.12.4-1.5.9</version> <classifier>${javacpp.platform.windows-x86_64}</classifier> </dependency> <!-- https://mvnrepository.com/artifact/org.bytedeco/librealsense2 --> <dependency> <groupId>org.bytedeco</groupId> <artifactId>librealsense2</artifactId> <version>2.53.1-1.5.9</version> <classifier>${javacpp.platform.windows-x86_64}</classifier> </dependency> <!-- https://mvnrepository.com/artifact/org.bytedeco/videoinput --> <dependency> <groupId>org.bytedeco</groupId> <artifactId>videoinput</artifactId> <version>0.200-1.5.9</version> <classifier>${javacpp.platform.windows-x86_64}</classifier> </dependency> <!-- https://mvnrepository.com/artifact/org.bytedeco/artoolkitplus --> <dependency> <groupId>org.bytedeco</groupId> <artifactId>artoolkitplus</artifactId> <version>2.3.1-1.5.9</version> <classifier>${javacpp.platform.windows-x86_64}</classifier> </dependency> </dependencies> </project>

2、代码

package ai.djl.examples.inference.cv; import ai.djl.Application; import ai.djl.Device; import ai.djl.MalformedModelException; import ai.djl.inference.Predictor; import ai.djl.modality.cv.BufferedImageFactory; import ai.djl.modality.cv.Image; import ai.djl.modality.cv.output.DetectedObjects; import ai.djl.modality.cv.translator.YoloV5Translator; import ai.djl.modality.cv.translator.YoloV8TranslatorFactory; import ai.djl.repository.zoo.Criteria; import ai.djl.repository.zoo.ModelNotFoundException; import ai.djl.repository.zoo.ZooModel; import ai.djl.training.util.ProgressBar; import ai.djl.translate.TranslateException; import ai.djl.translate.Translator; import org.bytedeco.ffmpeg.global.avutil; import org.bytedeco.javacv.Java2DFrameUtils; import org.bytedeco.javacv.*; import org.bytedeco.opencv.opencv_core.Mat; import org.opencv.core.Core; import org.opencv.core.MatOfPoint; import org.opencv.core.Scalar; import org.opencv.imgproc.Imgproc; import javax.swing.*; import java.awt.image.BufferedImage; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; import java.util.Objects; public class Rtsp { /** * 原始RTSP流地址 */ private static final String RTSP = "rtmp://rtmp02open.ys7.com:1935/v3/openlive/33010243992887901197:33010927991327431255_1_1?expire=1765510803&id=919539092630331392&t=16f218c3c40b7b26d9ec84316603bf9a2dc7fa677969701585cda33709331889&ev=101&devProto=gb28181&supportH265=1"; /** * 模型路径 */ private static final String path = "D:\\LIHAOWORK\\models\\yolov5-pt\\model\\person\\person.onnx"; /** * 围栏多边形顶点 */ private static final org.opencv.core.Point[] points = {new org.opencv.core.Point(0, 300), new org.opencv.core.Point(350, 340), new org.opencv.core.Point(400, 500), new org.opencv.core.Point(0, 720),}; private static Predictor<Image, DetectedObjects> predictor; private static DetectedObjects result; private static float threshold = 0.2f; /** * 视频帧率 */ private static int frameRate = 30; /** * 视频帧宽度 */ private static int width = 640; /** * 视频帧高度 */ private static int height = 640; /** * 初始化 */ private static void init() { Criteria<Image, DetectedObjects> criteria = Criteria.builder() // 设置输入输出 .setTypes(Image.class, DetectedObjects.class) // .optModelUrls("djl://ai.djl.pytorch/yolo11n") .optModelUrls("https://mlrepo.djl.ai/model/cv/object_detection/ai/djl/onnxruntime/yolo11n/0.0.1/yolo11n.zip") .optEngine("OnnxRuntime") .optArgument("width", 640) .optArgument("height", 640) .optArgument("resize", true) .optArgument("toTensor", true) .optArgument("applyRatio", true) .optArgument("threshold", 0.6f) // for performance optimization maxBox parameter can reduce number of // considered boxes from 8400 .optArgument("maxBox", 1000) .optTranslatorFactory(new YoloV8TranslatorFactory()) // 进度条 .optProgress(new ProgressBar()) .build(); try { //模型加载 ZooModel<Image, DetectedObjects> model = criteria.loadModel(); predictor = model.newPredictor(); System.out.println("模型加载完成"); // System.loadLibrary(Core.NATIVE_LIBRARY_NAME); System.out.println("底层库加载完成"); } catch (IOException e) { e.printStackTrace(); } catch (ModelNotFoundException e) { e.printStackTrace(); } catch (MalformedModelException e) { e.printStackTrace(); } } public static void main(String[] args) { //开始抽帧 System.out.println("开始抽帧"); FFmpegFrameGrabber grabber = null; try { grabber = new FFmpegFrameGrabber("f:\\2020.mp4"); // grabber.setOption("rtsp_transport", "tcp"); // 使用tcp的方式 // grabber.setOption("stimeout", "5000000"); grabber.setPixelFormat(avutil.AV_PIX_FMT_RGB24); // 像素格式 grabber.setImageWidth(width); grabber.setImageHeight(height); grabber.setFrameRate(frameRate); grabber.start(); //初始化模型 System.out.println("初始化模型"); init(); //播放窗口 System.out.println("播放窗口"); CanvasFrame canvasFrame = new CanvasFrame("摄像机"); canvasFrame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); canvasFrame.setAlwaysOnTop(true); //核心处理逻辑 System.out.println("核心处理逻辑"); //初始化计数器为0 int i = 0; while (true) { //测试环境下无线循环 //抽取一帧 Frame frame = grabber.grabFrame(); if (frame == null) { continue; } //转换、推理、识别、绘制、转换 frame = processFrame(frame, i); if (Objects.isNull(frame)) { continue; } //展示 canvasFrame.showImage(frame); //计数器累加 i++; if (i >= frameRate) { //当计数器大于等于帧率时候重置为0 i = 0; } } } catch (Exception e) { e.printStackTrace(); } finally { } } /** * 推理 * * @param frame * @param i * @return */ private static Frame processFrame(Frame frame, int i) { System.out.println("1(frame2Image)"); Long start1 = System.currentTimeMillis(); Image image = frame2Image(frame); if (Objects.isNull(image)) { return null; } Long end1 = System.currentTimeMillis(); System.out.println("frame2Image耗时:" + (end1 - start1) + "ms"); //3帧推理一帧 if (i % 3 == 0) { try { System.out.println("2(推理)"); Long start2 = System.currentTimeMillis(); result = predictor.predict(image); Long end2 = System.currentTimeMillis(); System.out.println("推理耗时:" + (end2 - start2) + "ms"); } catch (TranslateException e) { e.printStackTrace(); } } System.out.println("3(结果)"); System.out.println(result); System.out.println("4(绘制)"); Long start3 = System.currentTimeMillis(); image.drawBoundingBoxes(result); Long end3 = System.currentTimeMillis(); System.out.println("绘制耗时:" + (end3 - start3) + "ms"); System.out.println("5(image2Frame)"); Long start4 = System.currentTimeMillis(); try { saveBoundingBoxImage(image, result, i); } catch (IOException e) { throw new RuntimeException(e); } Mat mat = image2Mat(image); Frame frameout = mat2Frame(mat); Long end4 = System.currentTimeMillis(); System.out.println("image2Frame耗时:" + (end4 - start4) + "ms"); return frameout; } /** * frame2Image * * @param frame * @return */ private static Image frame2Image(Frame frame) { BufferedImage temp = Java2DFrameUtils.toBufferedImage(frame); if (Objects.isNull(temp)) { return null; } Image image = BufferedImageFactory.getInstance().fromImage(temp); return image; } private static BufferedImage frame2BufferedImage(Frame frame) { BufferedImage bufferedImage = Java2DFrameUtils.toBufferedImage(frame); return bufferedImage; } /** * image2Frame * * @param image * @return */ private static Frame image2Frame(Image image) { BufferedImage temp = (BufferedImage) image.getWrappedImage(); Frame frame = Java2DFrameUtils.toFrame(temp); return frame; } /** * image2Mat * * @param image * @return */ private static Mat image2Mat(Image image) { BufferedImage temp = (BufferedImage) image.getWrappedImage(); Mat mat = Java2DFrameUtils.toMat(temp); return mat; } /** * mat2Frame * * @param mat * @return */ private static Frame mat2Frame(Mat mat) { Frame frame = Java2DFrameUtils.toFrame(mat); return frame; } /** * 模拟绘制电子围栏 * * @param mat * @param points */ private static void drawRect(Mat mat, org.opencv.core.Point[] points) { OpenCVFrameConverter.ToMat converter1 = new OpenCVFrameConverter.ToMat(); OpenCVFrameConverter.ToOrgOpenCvCoreMat converter2 = new OpenCVFrameConverter.ToOrgOpenCvCoreMat(); org.opencv.core.Mat cvmat = converter2.convert(converter1.convert(mat)); MatOfPoint ps = new MatOfPoint(); ps.fromArray(points); //Scalar 颜色 Scalar scalar = new Scalar(255, 0, 255); Imgproc.polylines(cvmat, Arrays.asList(ps), true, scalar, 5, Imgproc.LINE_8); } private static void saveBoundingBoxImage(Image img, DetectedObjects detection, int i) throws IOException { Path outputDir = Paths.get("build/output"); Files.createDirectories(outputDir); img.drawBoundingBoxes(detection); Path imagePath = outputDir.resolve("rtsp" + i + ".png"); // OpenJDK can't save jpg with alpha channel img.save(Files.newOutputStream(imagePath), "png"); } }

3、运行效果