banboshi_V2/halftoneproject-master/BanBoShi/ImageProcessing/DefectLib.cs

//#define Defect3

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Drawing;
using System.Drawing.Imaging;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Forms;
using MaiMuAOI.SysCtrl;
using MaiMuControl.Device;
using Microsoft.ML.OnnxRuntime;
using Microsoft.ML.OnnxRuntime.Tensors;
using OpenCvSharp;
using Yolo5;

namespace MaiMuAOI.ImageProcessing
{
    public class DefectLib : IDisposable
    {
        public Action<WarningEnum, string> WarningEvent;
        /// <summary>
        /// 检测结果JSON(原图,结果)
        /// </summary>
        public Action<DefectTask> finishEvent;
        /// <summary>
        /// 是否打开设备成功
        /// </summary>
        public bool IsInit { get; private set; } = false;
        //private System.Timers.Timer timer = new System.Timers.Timer();
        private Yolo_Class yolo1;
        private InferenceSession _onnxSession;
        private Yolo_Class yolo2;
        private InferenceSession _onnxSession2;
        private Yolo_Class yolo3;
        private InferenceSession _onnxSession3;
        // 
        private Thread t_task, t_task_operation, t_task_operation2, t_task_operation3, t_task_tag;
        //=======task list
        private List<DefectTask> taskList = new List<DefectTask>(); 
        private List<DefectTask> taskOperationList = new List<DefectTask>();
        private List<DefectTask> taskTagList = new List<DefectTask>();

        public DefectLib()
        {
        } 
        public bool start()
        {
            try
            {
                yolo1 = new Yolo_Class();
                yolo2 = new Yolo_Class();
#if Defect3
                yolo3 = new Yolo_Class();
#endif
                //加载模型（只加载一次即可）
                //string modelFilePath = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) + "\\DevCfg\\best.onnx";
                //_onnxSession = yolo1.LoadModel(modelFilePath, true);//false-CPU true-显卡

                IsInit = true;
                //timer.Elapsed += Timer_Elapsed;
                //timer.Interval = 100;
                //timer.Enabled = true;
                taskList.Clear(); 
                taskOperationList.Clear();
                taskTagList.Clear();

                //MessageBox.Show("11");
                t_task = new System.Threading.Thread(runStep);
                t_task.IsBackground = true;
                t_task.Start();

                //MessageBox.Show("22");
                t_task_operation = new System.Threading.Thread(run2);
                t_task_operation.IsBackground = true;
                t_task_operation.Start();

                //MessageBox.Show("33");
                t_task_operation2 = new System.Threading.Thread(run3);
                t_task_operation2.IsBackground = true;
                t_task_operation2.Start();

                //MessageBox.Show("44");
#if Defect3
                t_task_operation3 = new System.Threading.Thread(run4);
                t_task_operation3.IsBackground = true;
                t_task_operation3.Start();
#endif
                //MessageBox.Show("55");
                return true;
            }
            catch (Exception ex)
            {
                WarningEvent?.Invoke(WarningEnum.High, ex.Message);
                return false;
            }
        }
        public void stop()
        {
            if (!IsInit) return;

            try
            {
                IsInit = false;
                //timer.Elapsed -= Timer_Elapsed;
                //释放模型
                if (_onnxSession != null)
                {
                    _onnxSession.Dispose();
                    _onnxSession = null;
                }
                if (_onnxSession2 != null)
                {
                    _onnxSession2.Dispose();
                    _onnxSession2 = null;
                }
#if Defect3
                if (_onnxSession3 != null)
                {
                    _onnxSession3.Dispose();
                    _onnxSession3 = null;
                }
#endif
                _preLoadONNXFilePath = null;

                if (t_task != null)
                {
                    bool b = t_task.Join(5000);
                    if (!b) t_task.Abort();
                    t_task = null;
                }

                if (t_task_operation != null)
                {
                    bool b = t_task_operation.Join(5000);
                    if (!b) t_task_operation.Abort();
                    t_task_operation = null;
                }
                if (t_task_operation2 != null)
                {
                    bool b = t_task_operation2.Join(5000);
                    if (!b) t_task_operation2.Abort();
                    t_task_operation2 = null;
                }
#if Defect3
                if (t_task_operation3 != null)
                {
                    bool b = t_task_operation3.Join(5000);
                    if (!b) t_task_operation3.Abort();
                    t_task_operation3 = null;
                }
#endif
                if (t_task_tag != null)
                {
                    bool b = t_task_tag.Join(5000);
                    if (!b) t_task_tag.Abort();
                    t_task_tag = null;
                }
                taskList.Clear(); 
                taskOperationList.Clear();
                taskTagList.Clear();
            }
            catch { }
        }
        private string _preLoadONNXFilePath;//记录上次成功加载的模型，不重复加载
        public void loadModelFile(string onnxFilePath)
        {
            try
            {
                //string modelFilePath = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) + "\\DevCfg\\best.onnx";
                //不重复加载
                if (!string.IsNullOrWhiteSpace(_preLoadONNXFilePath) && _preLoadONNXFilePath.ToLower() == onnxFilePath.ToLower())
                    return;

                //MessageBox.Show("1");
                //Task.Run(async () =>
                //{
                //    if (_onnxSession != null)
                //    {
                //        _onnxSession.Dispose();
                //        _onnxSession = null;
                //    }
                //    _onnxSession = yolo1.LoadModel(onnxFilePath, true);//false-CPU true-显卡
                //    _preLoadONNXFilePath = onnxFilePath;
                //});
                if (_onnxSession != null)
                {
                    _onnxSession.Dispose();
                    _onnxSession = null;
                }
                _onnxSession = yolo1.LoadModel(onnxFilePath, true);//false-CPU true-显卡

                //MessageBox.Show("2");
                if (_onnxSession2 != null)
                {
                    _onnxSession2.Dispose();
                    _onnxSession2 = null;
                }
                _onnxSession2 = yolo2.LoadModel(onnxFilePath, true);//false-CPU true-显卡
#if Defect3
                //MessageBox.Show("3");
                if (_onnxSession3 != null)
                {
                    _onnxSession3.Dispose();
                    _onnxSession3 = null;
                }
                _onnxSession3 = yolo3.LoadModel(onnxFilePath, true);//false-CPU true-显卡
#endif
                //MessageBox.Show("4");
                _preLoadONNXFilePath = onnxFilePath;
            }
            catch (Exception ex)
            {
                MessageBox.Show(ex.Message);
            }
        }
        /// <summary>
        /// 保存图片
        /// </summary>
        /// <param name="Img"></param>图片
        /// <param name="pictureUrl"></param>保存路径
        /// <param name="pictureName"></param>保存名称
        private void SaveImage(Bitmap Img, string pictureUrl, string pictureName)
        {
            if (!Directory.Exists(pictureUrl))
            {
                Directory.CreateDirectory(pictureUrl);
            }
            FileInfo FileUrl = new FileInfo(pictureUrl);//防止路径中有日期导致路径错误

            try
            {
                using (Bitmap bitmap = new Bitmap(Img))
                {
                    using (MemoryStream stream = new MemoryStream())
                    {
                        bitmap.Save(stream, ImageFormat.Bmp);
                        System.Drawing.Image img = System.Drawing.Image.FromStream(stream);
                        //img.Save(FileUrl +"\\"+ pictureName);
                        //img.Dispose();
                        string szURL = FileUrl + "\\" + pictureName;
                        img.Save(szURL, ImageFormat.Bmp);
                        img.Dispose();
                    }
                }
            }
            catch (Exception)
            {
                if (Img != null)
                {
                    Img.Dispose();
                }
            }
        }
        private void runStep()
        {
            while (IsInit)
            {
                if (taskList.Count < 1 || _onnxSession == null)
                {
                    Thread.Sleep(5);
                    continue;
                }

                //
                var task = pop(); 
                try
                {
                    if (task != null)
                    {
                        Stopwatch stopwatch = Stopwatch.StartNew();
                        //源图  
                        //Bitmap bmp = yolo1.Read2Bmp(file_path);
                        //切割图像，输入图像格式14208*10640
                        stopwatch.Start();
                        task.bmps_cut = yolo1.OpenCVToCutsMat(task.bmp, task.cut_size.Width, task.cut_size.Height);
                        stopwatch.Stop();
                        task.stopwatch[0] = stopwatch.ElapsedMilliseconds;

                        //Resize图像
                        stopwatch.Restart();
                        task.bmps_resize = yolo1.OpenCVToResizesMat(task.bmps_cut, task.resize.Width, task.resize.Height);
                        stopwatch.Stop();
                        task.stopwatch[1] = stopwatch.ElapsedMilliseconds;

                        //预处理模型
                        stopwatch.Restart();
                        task.tensors = yolo1.PreprocessImageMat(task.bmps_resize);
                        stopwatch.Stop();
                        task.stopwatch[2] = stopwatch.ElapsedMilliseconds;

                        lock (taskOperationList)
                            taskOperationList.Add(task);
                    }
                    Thread.Sleep(5);
                }
                catch (Exception ex)
                {
                    WarningEvent?.Invoke(WarningEnum.Low, "DefectLib task1 err:" + ex.Message);
                    task.isSucceed = false;
                    task.resultInfo = ex.Message;
                    callback(task);
                } 
            }
        }  

        private void run2()
        {
            while (IsInit)
            {
                if (taskOperationList.Count < 1)
                {
                    Thread.Sleep(5);
                    continue;
                }

                //
                var task = pop2();
                int liStep = 0;
                try
                {
                    if (task != null)
                    {
                        Stopwatch stopwatch = Stopwatch.StartNew();
                        //====运行推理（必需单队列）
                        stopwatch.Start();
                        IDisposableReadOnlyCollection<DisposableNamedOnnxValue>[] results = yolo1.RunModlel(_onnxSession, task.tensors);
                        liStep = 1;
                        if (task.modelType == "rj")
                            task.informationList = yolo1.ScreeningResults_YD_RJ(results, task.bmps_resize, task.thresholds, task.thresholdsClass, task.recAreaThreshold);
                        else
                            task.informationList = yolo1.ScreeningResults_YD(results, task.bmps_resize, task.thresholds, task.thresholdsClass, task.recAreaThreshold);
                        liStep = 2;
                         
                        //当前大图上缺陷个数
                        int currPicDefectCount = 0;
                        for (int x = 0; x < task.informationList.Count(); x++)
                            currPicDefectCount += task.informationList[x].Count();
                        task.defectCount = currPicDefectCount;
                        stopwatch.Stop();
                        task.stopwatch[3] = stopwatch.ElapsedMilliseconds;
                        liStep = 3;
                        if (task.informationList.Count > 0)
                        {
                            liStep = 4;
                            //lock (taskTagList)
                            //    taskTagList.Add(task);
                            //====打标 ,有缺陷返回缺陷图片数组，没缺陷返回所有图片数组 
                            stopwatch.Restart();
                            task.bmps_tag = yolo1.DrawYoloPrediction2Mat(task.bmps_cut, task.informationList, 27, SixLabors.ImageSharp.Color.OrangeRed, task.resize.Width,
                                task.Xmm, task.Ymm, out task.defectInfor2RestorationDesk);

                            liStep = 5;
                            if (ConfMgr.Instance.SysConfigParams.OpenFlawDistribution)
                            {
                                //大图缺陷坐标转换到图纸坐标 
                                if (!string.IsNullOrWhiteSpace(task.drawingPagePath) && task.defectInfor2RestorationDesk != null && task.defectInfor2RestorationDesk.Count > 0)
                                    task.defectInfor2RestorationDeskPage = yolo1.DefectDrawGerberImage(task.drawingPagePath, task.defectInfor2RestorationDesk);
                            }
                            stopwatch.Stop();
                            task.stopwatch[4] = stopwatch.ElapsedMilliseconds;

                            liStep = 6;
                            task.bmpCompress = yolo1.ResizesMat_4(task.bmp);
                            task.isSucceed = true;
                            callback(task);
                        }
                        else
                        { 
                            task.isSucceed = true;
                            callback(task);
                        }
                    }
                    Thread.Sleep(5);
                }
                catch (Exception ex)
                {
                    WarningEvent?.Invoke(WarningEnum.Low, $"DefectLib task2 err({liStep}):" + ex.Message);
                    task.isSucceed = false;
                    task.resultInfo = ex.Message;
                    callback(task);
                } 
            }
        }

        private void run3()
        {
            while (IsInit)
            {
                if (taskOperationList.Count < 1)
                {
                    Thread.Sleep(5);
                    continue;
                }

                //
                var task = pop2();
                int liStep = 0;
                try
                {
                    if (task != null)
                    {
                        Stopwatch stopwatch = Stopwatch.StartNew();
                        //====运行推理（必需单队列）
                        stopwatch.Start();
                        IDisposableReadOnlyCollection<DisposableNamedOnnxValue>[] results = yolo2.RunModlel(_onnxSession2, task.tensors);
                        liStep = 1;
                        if (task.modelType == "rj")
                            task.informationList = yolo2.ScreeningResults_YD_RJ(results, task.bmps_resize, task.thresholds, task.thresholdsClass, task.recAreaThreshold);
                        else
                            task.informationList = yolo2.ScreeningResults_YD(results, task.bmps_resize, task.thresholds, task.thresholdsClass, task.recAreaThreshold);
                        liStep = 2;

                        //当前大图上缺陷个数
                        int currPicDefectCount = 0;
                        for (int x = 0; x < task.informationList.Count(); x++)
                            currPicDefectCount += task.informationList[x].Count();
                        task.defectCount = currPicDefectCount;
                        stopwatch.Stop();
                        task.stopwatch[3] = stopwatch.ElapsedMilliseconds;
                        liStep = 3;
                        if (task.informationList.Count > 0)
                        {
                            liStep = 4;
                            //lock (taskTagList)
                            //    taskTagList.Add(task);
                            //====打标 ,有缺陷返回缺陷图片数组，没缺陷返回所有图片数组 
                            stopwatch.Restart();
                            task.bmps_tag = yolo2.DrawYoloPrediction2Mat(task.bmps_cut, task.informationList, 27, SixLabors.ImageSharp.Color.OrangeRed, task.resize.Width,
                                task.Xmm, task.Ymm, out task.defectInfor2RestorationDesk);

                            liStep = 5;
                            if (ConfMgr.Instance.SysConfigParams.OpenFlawDistribution)
                            {
                                //大图缺陷坐标转换到图纸坐标 
                                if (!string.IsNullOrWhiteSpace(task.drawingPagePath) && task.defectInfor2RestorationDesk != null && task.defectInfor2RestorationDesk.Count > 0)
                                    task.defectInfor2RestorationDeskPage = yolo2.DefectDrawGerberImage(task.drawingPagePath, task.defectInfor2RestorationDesk);
                            }
                            stopwatch.Stop();
                            task.stopwatch[4] = stopwatch.ElapsedMilliseconds;

                            liStep = 6;
                            task.bmpCompress = yolo2.ResizesMat_4(task.bmp);
                            task.isSucceed = true;
                            callback(task);
                        }
                        else
                        {
                            task.isSucceed = true;
                            callback(task);
                        }
                    }
                    Thread.Sleep(5);
                }
                catch (Exception ex)
                {
                    WarningEvent?.Invoke(WarningEnum.Low, $"DefectLib task3 err({liStep}):" + ex.Message);
                    task.isSucceed = false;
                    task.resultInfo = ex.Message;
                    callback(task);
                }
            }
        }

        private void run4()
        {
            while (IsInit)
            {
                if (taskOperationList.Count < 1)
                {
                    Thread.Sleep(5);
                    continue;
                }

                //
                var task = pop2();
                int liStep = 0;
                try
                {
                    if (task != null)
                    {
                        Stopwatch stopwatch = Stopwatch.StartNew();
                        //====运行推理（必需单队列）
                        stopwatch.Start();
                        IDisposableReadOnlyCollection<DisposableNamedOnnxValue>[] results = yolo3.RunModlel(_onnxSession3, task.tensors);
                        liStep = 1;
                        if(task.modelType == "rj")
                            task.informationList = yolo3.ScreeningResults_YD_RJ(results, task.bmps_resize, task.thresholds, task.thresholdsClass, task.recAreaThreshold);
                        else
                            task.informationList = yolo3.ScreeningResults_YD(results, task.bmps_resize, task.thresholds, task.thresholdsClass, task.recAreaThreshold);
                        liStep = 2;

                        //当前大图上缺陷个数
                        int currPicDefectCount = 0;
                        for (int x = 0; x < task.informationList.Count(); x++)
                            currPicDefectCount += task.informationList[x].Count();
                        task.defectCount = currPicDefectCount;
                        stopwatch.Stop();
                        task.stopwatch[3] = stopwatch.ElapsedMilliseconds;
                        liStep = 3;
                        if (task.informationList.Count > 0)
                        {
                            liStep = 4;
                            //lock (taskTagList)
                            //    taskTagList.Add(task);
                            //====打标 ,有缺陷返回缺陷图片数组，没缺陷返回所有图片数组 
                            stopwatch.Restart();
                            task.bmps_tag = yolo3.DrawYoloPrediction2Mat(task.bmps_cut, task.informationList, 27, SixLabors.ImageSharp.Color.OrangeRed, task.resize.Width,
                                task.Xmm, task.Ymm, out task.defectInfor2RestorationDesk);

                            liStep = 5;
                            if (ConfMgr.Instance.SysConfigParams.OpenFlawDistribution)
                            {
                                //大图缺陷坐标转换到图纸坐标 
                                if (!string.IsNullOrWhiteSpace(task.drawingPagePath) && task.defectInfor2RestorationDesk != null && task.defectInfor2RestorationDesk.Count > 0)
                                    task.defectInfor2RestorationDeskPage = yolo3.DefectDrawGerberImage(task.drawingPagePath, task.defectInfor2RestorationDesk);
                            }
                            stopwatch.Stop();
                            task.stopwatch[4] = stopwatch.ElapsedMilliseconds;

                            liStep = 6;
                            task.bmpCompress = yolo3.ResizesMat_4(task.bmp);
                            task.isSucceed = true;
                            callback(task);
                        }
                        else
                        {
                            task.isSucceed = true;
                            callback(task);
                        }
                    }
                    Thread.Sleep(5);
                }
                catch (Exception ex)
                {
                    WarningEvent?.Invoke(WarningEnum.Low, $"DefectLib task4 err({liStep}):" + ex.Message);
                    task.isSucceed = false;
                    task.resultInfo = ex.Message;
                    callback(task);
                }
            }
        }
        /// <summary>
        /// 已不用
        /// </summary>
        private void run5()
        {
            while (IsInit)
            {
                if (taskTagList.Count < 1)
                {
                    Thread.Sleep(0);
                    continue;
                }

                //
                var task = pop3(); 
                try
                {
                    if (task != null)
                    {
                        Stopwatch stopwatch = Stopwatch.StartNew();
                        //====打标 ,有缺陷返回缺陷图片数组，没缺陷返回所有图片数组
                        stopwatch.Start();
                        task.bmps_tag = yolo1.DrawYoloPrediction2Mat(task.bmps_cut, task.informationList, 27, SixLabors.ImageSharp.Color.OrangeRed, task.resize.Width,
                            task.Xmm,task.Ymm,out task.defectInfor2RestorationDesk);          
                        stopwatch.Stop();
                        task.stopwatch[4] = stopwatch.ElapsedMilliseconds;
                        task.isSucceed = true;
                        callback(task);
                    }
                }
                catch (Exception ex)
                {
                    WarningEvent?.Invoke(WarningEnum.Low, "DefectLib task3 err:" + ex.Message);
                    task.isSucceed = false;
                    task.resultInfo = ex.Message;
                    callback(task);
                } 
            }
        }
        private void callback(DefectTask task)
        {
            //返回成功/失败,异步调用 
            if (task.finishEvent != null || (task.finishEvent = finishEvent) != null)
                //task.finishEvent.BeginInvoke(result, errInfo, res => task.finishEvent.EndInvoke(res), null);
                System.Threading.ThreadPool.QueueUserWorkItem(waitCallback, task);
        }
        //异步回调
        WaitCallback waitCallback = new WaitCallback(o =>
        {
            var task = (DefectTask)o;
            task.finishEvent(task);
        });
        
        public class DefectTask
        {
            public int stepIndex;
            public string processName;

            //2024 03 22 加入模型类型 rj pt;
            public string modelType;

            public string drawingPagePath;//图纸路径bmp
            //图索引
            public int index = 0;
            /// <summary>
            /// 绝对位置-----X2轴
            /// </summary>
            public double Xmm;
            /// <summary>
            /// 绝对位置-----Y轴
            /// </summary>
            public double Ymm;
            /// <summary>
            /// 源文件
            /// </summary>
            public Mat bmp;  
            public Mat bmpCompress;//压缩有缺陷大图
            public System.Drawing.Size cut_size = new System.Drawing.Size(592, 532);
            public System.Drawing.Size resize = new System.Drawing.Size(224, 224);
            /// <summary>
            /// 阈值
            /// </summary>
            public float thresholds = 0.4f;
            /// <summary>
            /// 种类阈值
            /// </summary>
            public string thresholdsClass;
            /// <summary>
            /// 缺陷在面积内数量
            /// </summary>
            public Dictionary<string, float> recAreaThreshold;
            /// <summary>
            /// 完成后回调 
            /// </summary>
            public Action<DefectTask> finishEvent;

            //
            public long createTime = DateTime.Now.Ticks;
            public DateTime nowTime;
            //中间变量，供step2使用
            public Mat[] bmps_cut;
            public Mat[] bmps_resize;
            //预处理模型
            public Tensor<float>[] tensors;

            //==结果返回
            public List<Dictionary<int, List<string>[]>> informationList;
            public int defectCount;//informationList中的缺陷数
            
            public List<List<string>> defectInfor2RestorationDesk, defectInfor2RestorationDeskPage;//打标缺陷转为图纸的坐标
            public Bitmap[] bmps_tag;  //打标小图 //bmps_tag.count<=bmps_cut.count bmps_tag.count=informationList.count
            public bool isSucceed;//转换是否成功
            public string resultInfo = "";//成功或失败信息
            public long[] stopwatch = new long[5];
        }

        public void add(DefectTask task)
        {
            lock (taskList)
                taskList.Add(task);
        }
        /// <summary>
        /// 打标 返回每张图的绝对位置X,Y
        /// </summary>
        /// <param name="defectBmpIndex">缺陷小图索引</param>
        /// <param name="X">拍照位绝对位置-----X2轴</param>
        /// <param name="Y">拍照位绝对位置-----Y轴</param>
        /// <param name="imagewidth">裁剪小图的宽592</param>
        /// <param name="imageheight">裁剪小图的高532</param>
        public List<double> makeTag(int defectBmpIndex, double X, double Y, int imagewidth, int imageheight)
        {
            return yolo1.ImageXY(defectBmpIndex, X, Y, imagewidth, imageheight);
        }
        /// <summary>
        /// 用相机1查看缺陷点
        /// </summary>
        /// <param name="defectBmpIndex"></param>
        /// <param name="X"></param>
        /// <param name="Y"></param>
        /// <param name="imagewidth"></param>
        /// <param name="imageheight"></param>
        /// <returns></returns>
        public List<double> viewTag(int defectBmpIndex, double X, double Y, int imagewidth, int imageheight)
        {
            return yolo1.ImageXY2(defectBmpIndex, X, Y, imagewidth, imageheight);
        }
        private DefectTask pop()
        {
            lock (taskList)
            {
                if (taskList.Count < 1)
                    return null;

                //int index = 0;// taskList.FindIndex(p => { return p.isSync; });
                //if (index < 0) index = 0; 
                var task = taskList[0];
                taskList.RemoveAt(0);
                return task;
            }
        }
        private DefectTask pop2()
        {
            lock (taskOperationList)
            {
                if (taskOperationList.Count < 1)
                    return null;

                //int index = 0;// taskList.FindIndex(p => { return p.isSync; });
                //if (index < 0) index = 0; 
                var task = taskOperationList[0];
                taskOperationList.RemoveAt(0);
                return task;
            }
        }
        private DefectTask pop3()
        {
            lock (taskTagList)
            {
                if (taskTagList.Count < 1)
                    return null;

                //int index = 0;// taskList.FindIndex(p => { return p.isSync; });
                //if (index < 0) index = 0; 
                var task = taskTagList[0];
                taskTagList.RemoveAt(0);
                return task;
            }
        } 

        public void Dispose()
        {
            stop();
        }
    }
}