//#define Defect3 //#define OpenSizeDefectComp using System; using System.Collections.Generic; using System.ComponentModel; using System.Diagnostics; using System.Drawing; using System.Drawing.Imaging; using System.IO; using System.Linq; using System.Reflection; using System.Threading; using System.Threading.Tasks; using System.Windows.Forms; using Microsoft.ML.OnnxRuntime; using Microsoft.ML.OnnxRuntime.Tensors; using Newtonsoft.Json; using OpenCvSharp; using Yolo5; namespace MaiMuAOI.ImageProcessing { public enum WarningEnum { [Description("正常")] Normal, [Description("低(可继续)")] Low, [Description("严重")] High } public class DefectLib : IDisposable { public Action WarningEvent; /// /// 检测结果JSON(原图,结果) /// public Action finishEvent; /// /// 是否打开设备成功 /// public bool IsInit { get; private set; } = false; //private System.Timers.Timer timer = new System.Timers.Timer(); private Yolo_Class yolo1; private InferenceSession _onnxSession; private Yolo_Class yolo2; private InferenceSession _onnxSession2; private Yolo_Class yolo3; private InferenceSession _onnxSession3; // private Thread t_task, t_task_operation, t_task_operation2, t_task_operation3, t_task_tag; //=======task list private List taskList = new List(); private List taskOperationList = new List(); private List taskTagList = new List(); public DefectLib() { } public bool start() { try { yolo1 = new Yolo_Class(); yolo2 = new Yolo_Class(); #if Defect3 yolo3 = new Yolo_Class(); #endif //加载模型(只加载一次即可) //string modelFilePath = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) + "\\DevCfg\\best.onnx"; //_onnxSession = yolo1.LoadModel(modelFilePath, true);//false-CPU true-显卡 IsInit = true; //timer.Elapsed += Timer_Elapsed; //timer.Interval = 100; //timer.Enabled = true; taskList.Clear(); taskOperationList.Clear(); taskTagList.Clear(); //MessageBox.Show("11"); t_task = new System.Threading.Thread(runStep); t_task.IsBackground = true; t_task.Start(); //MessageBox.Show("22"); t_task_operation = new System.Threading.Thread(run2); t_task_operation.IsBackground = true; t_task_operation.Start(); //MessageBox.Show("33"); t_task_operation2 = new System.Threading.Thread(run3); t_task_operation2.IsBackground = true; t_task_operation2.Start(); //MessageBox.Show("44"); #if Defect3 t_task_operation3 = new System.Threading.Thread(run4); t_task_operation3.IsBackground = true; t_task_operation3.Start(); #endif //MessageBox.Show("55"); return true; } catch (Exception ex) { WarningEvent?.Invoke(WarningEnum.High, ex.Message); return false; } } public void stop() { if (!IsInit) return; try { IsInit = false; //timer.Elapsed -= Timer_Elapsed; //释放模型 if (_onnxSession != null) { _onnxSession.Dispose(); _onnxSession = null; } if (_onnxSession2 != null) { _onnxSession2.Dispose(); _onnxSession2 = null; } #if Defect3 if (_onnxSession3 != null) { _onnxSession3.Dispose(); _onnxSession3 = null; } #endif _preLoadONNXFilePath = null; if (t_task != null) { bool b = t_task.Join(5000); if (!b) t_task.Abort(); t_task = null; } if (t_task_operation != null) { bool b = t_task_operation.Join(5000); if (!b) t_task_operation.Abort(); t_task_operation = null; } if (t_task_operation2 != null) { bool b = t_task_operation2.Join(5000); if (!b) t_task_operation2.Abort(); t_task_operation2 = null; } #if Defect3 if (t_task_operation3 != null) { bool b = t_task_operation3.Join(5000); if (!b) t_task_operation3.Abort(); t_task_operation3 = null; } #endif if (t_task_tag != null) { bool b = t_task_tag.Join(5000); if (!b) t_task_tag.Abort(); t_task_tag = null; } taskList.Clear(); taskOperationList.Clear(); taskTagList.Clear(); } catch { } } private string _preLoadONNXFilePath;//记录上次成功加载的模型,不重复加载 public void loadModelFile(string onnxFilePath, bool UseGPU) { try { //string modelFilePath = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) + "\\DevCfg\\best.onnx"; //不重复加载 if (!string.IsNullOrWhiteSpace(_preLoadONNXFilePath) && _preLoadONNXFilePath.ToLower() == onnxFilePath.ToLower()) return; //MessageBox.Show("1"); //Task.Run(async () => //{ // if (_onnxSession != null) // { // _onnxSession.Dispose(); // _onnxSession = null; // } // _onnxSession = yolo1.LoadModel(onnxFilePath, true);//false-CPU true-显卡 // _preLoadONNXFilePath = onnxFilePath; //}); if (_onnxSession != null) { _onnxSession.Dispose(); _onnxSession = null; } _onnxSession = yolo1.LoadModel(onnxFilePath, UseGPU);//false-CPU true-显卡 //MessageBox.Show("2"); if (_onnxSession2 != null) { _onnxSession2.Dispose(); _onnxSession2 = null; } _onnxSession2 = yolo2.LoadModel(onnxFilePath, UseGPU);//false-CPU true-显卡 #if Defect3 //MessageBox.Show("3"); if (_onnxSession3 != null) { _onnxSession3.Dispose(); _onnxSession3 = null; } _onnxSession3 = yolo3.LoadModel(onnxFilePath, true);//false-CPU true-显卡 #endif //MessageBox.Show("4"); _preLoadONNXFilePath = onnxFilePath; } catch (Exception ex) { MessageBox.Show(ex.Message); } } /// /// 保存图片 /// /// 图片 /// 保存路径 /// 保存名称 private void SaveImage(Bitmap Img, string pictureUrl, string pictureName) { if (!Directory.Exists(pictureUrl)) { Directory.CreateDirectory(pictureUrl); } FileInfo FileUrl = new FileInfo(pictureUrl);//防止路径中有日期导致路径错误 try { using (Bitmap bitmap = new Bitmap(Img)) { using (MemoryStream stream = new MemoryStream()) { bitmap.Save(stream, ImageFormat.Bmp); System.Drawing.Image img = System.Drawing.Image.FromStream(stream); //img.Save(FileUrl +"\\"+ pictureName); //img.Dispose(); string szURL = FileUrl + "\\" + pictureName; img.Save(szURL, ImageFormat.Bmp); img.Dispose(); } } } catch (Exception) { if (Img != null) { Img.Dispose(); } } } private void runStep() { while (IsInit) { if (taskList.Count < 1 || _onnxSession == null) { Thread.Sleep(5); continue; } // var task = pop(); try { if (task != null) { Stopwatch stopwatch = Stopwatch.StartNew(); //源图 //Bitmap bmp = yolo1.Read2Bmp(file_path); //切割图像,输入图像格式14208*10640 stopwatch.Start(); task.bmps_cut = yolo1.OpenCVToCutsMat(task.bmp, task.cut_size.Width, task.cut_size.Height); stopwatch.Stop(); task.stopwatch[0] = stopwatch.ElapsedMilliseconds; //Resize图像 stopwatch.Restart(); task.bmps_resize = yolo1.OpenCVToResizesMat(task.bmps_cut, task.resize.Width, task.resize.Height); stopwatch.Stop(); task.stopwatch[1] = stopwatch.ElapsedMilliseconds; //预处理模型 stopwatch.Restart(); task.tensors = yolo1.PreprocessImageMat(task.bmps_resize); stopwatch.Stop(); task.stopwatch[2] = stopwatch.ElapsedMilliseconds; lock (taskOperationList) taskOperationList.Add(task); } Thread.Sleep(5); } catch (Exception ex) { WarningEvent?.Invoke(WarningEnum.Low, "DefectLib task1 err:" + ex.Message); task.isSucceed = false; task.resultInfo = ex.Message; callback(task); } } } private void run2() { while (IsInit) { if (taskOperationList.Count < 1) { Thread.Sleep(5); continue; } // var task = pop2(); int liStep = 0; try { if (task != null) { Stopwatch stopwatch = Stopwatch.StartNew(); //====运行推理(必需单队列) stopwatch.Start(); IDisposableReadOnlyCollection[] results = yolo1.RunModlel(_onnxSession, task.tensors); liStep = 1; if (task.modelType == "rj") task.informationList = yolo1.ScreeningResults_YD_RJ(results, task.bmps_resize, task.thresholds, task.thresholdsClass, task.recAreaThreshold); else task.informationList = yolo1.ScreeningResults_YD(results, task.bmps_resize, task.thresholds, task.thresholdsClass, task.recAreaThreshold); liStep = 2; //当前大图上缺陷个数 int currPicDefectCount = 0; for (int x = 0; x < task.informationList.Count(); x++) currPicDefectCount += task.informationList[x].Count(); task.defectCount = currPicDefectCount; stopwatch.Stop(); task.stopwatch[3] = stopwatch.ElapsedMilliseconds; liStep = 3; if (task.informationList.Count > 0) { liStep = 4; //lock (taskTagList) // taskTagList.Add(task); //====打标 ,有缺陷返回缺陷图片数组,没缺陷返回所有图片数组 stopwatch.Restart(); #if OpenSizeDefectComp task.bmps_tag = yolo1.DrawYoloPrediction2Mat(task.bmps_cut, task.informationList, 27, SixLabors.ImageSharp.Color.OrangeRed, task.resize.Width, task.Xmm, task.Ymm, out task.defectInfor2RestorationDesk, out task.defectInfor2BigImage); #else task.bmps_tag = yolo1.DrawYoloPrediction2Mat(task.bmps_cut, task.informationList, 27, SixLabors.ImageSharp.Color.OrangeRed, task.resize.Width, task.Xmm, task.Ymm, out task.defectInfor2RestorationDesk); #endif liStep = 5; //List> SizedefectInfor2RestorationDesk = SysMgr.Instance.GetSizeDefectData(task.Xmm, task.Ymm); //task.defectInfor2RestorationDesk = yolo3.FilteringOverlappingDefects(task.defectInfor2RestorationDesk, SizedefectInfor2RestorationDesk); //liStep = 51; //if (ConfMgr.Instance.SysConfigParams.OpenFlawDistribution) //{ // //大图缺陷坐标转换到图纸坐标 // if (!string.IsNullOrWhiteSpace(task.drawingPagePath) && task.defectInfor2RestorationDesk != null && task.defectInfor2RestorationDesk.Count > 0) // task.defectInfor2RestorationDeskPage = yolo1.DefectDrawGerberImage(task.drawingPagePath, task.defectInfor2RestorationDesk); //} stopwatch.Stop(); task.stopwatch[4] = stopwatch.ElapsedMilliseconds; liStep = 6; task.bmpCompress = yolo1.ResizesMat_4(task.bmp); //if (!ConfMgr.Instance.SysConfigParams.OpenColorB) // task.bmpCompress = yolo1.ResizesMat_4(task.bmp); //else // task.bmpCompress = yolo1.ResizesMat_4(task.srcbmp); task.isSucceed = true; callback(task); } else { task.isSucceed = true; callback(task); } } Thread.Sleep(5); } catch (Exception ex) { WarningEvent?.Invoke(WarningEnum.Low, $"DefectLib task2 err({liStep}):" + ex.Message + $"||{task.modelType}-{task.thresholds}, {task.thresholdsClass}, {JsonConvert.SerializeObject(task.recAreaThreshold)}"); task.isSucceed = false; task.resultInfo = ex.Message; callback(task); } } } private void run3() { while (IsInit) { if (taskOperationList.Count < 1) { Thread.Sleep(5); continue; } // var task = pop2(); int liStep = 0; try { if (task != null) { Stopwatch stopwatch = Stopwatch.StartNew(); //====运行推理(必需单队列) stopwatch.Start(); IDisposableReadOnlyCollection[] results = yolo2.RunModlel(_onnxSession2, task.tensors); liStep = 1; if (task.modelType == "rj") task.informationList = yolo2.ScreeningResults_YD_RJ(results, task.bmps_resize, task.thresholds, task.thresholdsClass, task.recAreaThreshold); else task.informationList = yolo2.ScreeningResults_YD(results, task.bmps_resize, task.thresholds, task.thresholdsClass, task.recAreaThreshold); liStep = 2; //当前大图上缺陷个数 int currPicDefectCount = 0; for (int x = 0; x < task.informationList.Count(); x++) currPicDefectCount += task.informationList[x].Count(); task.defectCount = currPicDefectCount; stopwatch.Stop(); task.stopwatch[3] = stopwatch.ElapsedMilliseconds; liStep = 3; if (task.informationList.Count > 0) { liStep = 4; //lock (taskTagList) // taskTagList.Add(task); //====打标 ,有缺陷返回缺陷图片数组,没缺陷返回所有图片数组 stopwatch.Restart(); #if OpenSizeDefectComp task.bmps_tag = yolo2.DrawYoloPrediction2Mat(task.bmps_cut, task.informationList, 27, SixLabors.ImageSharp.Color.OrangeRed, task.resize.Width, task.Xmm, task.Ymm, out task.defectInfor2RestorationDesk, out task.defectInfor2BigImage); #else task.bmps_tag = yolo2.DrawYoloPrediction2Mat(task.bmps_cut, task.informationList, 27, SixLabors.ImageSharp.Color.OrangeRed, task.resize.Width, task.Xmm, task.Ymm, out task.defectInfor2RestorationDesk); #endif liStep = 5; //List> SizedefectInfor2RestorationDesk = SysMgr.Instance.GetSizeDefectData(task.Xmm, task.Ymm); //task.defectInfor2RestorationDesk = yolo3.FilteringOverlappingDefects(task.defectInfor2RestorationDesk, SizedefectInfor2RestorationDesk); //liStep = 51; //if (ConfMgr.Instance.SysConfigParams.OpenFlawDistribution) //{ // //大图缺陷坐标转换到图纸坐标 // if (!string.IsNullOrWhiteSpace(task.drawingPagePath) && task.defectInfor2RestorationDesk != null && task.defectInfor2RestorationDesk.Count > 0) // task.defectInfor2RestorationDeskPage = yolo2.DefectDrawGerberImage(task.drawingPagePath, task.defectInfor2RestorationDesk); //} stopwatch.Stop(); task.stopwatch[4] = stopwatch.ElapsedMilliseconds; liStep = 6; task.bmpCompress = yolo2.ResizesMat_4(task.bmp); task.isSucceed = true; callback(task); } else { task.isSucceed = true; callback(task); } } Thread.Sleep(5); } catch (Exception ex) { WarningEvent?.Invoke(WarningEnum.Low, $"DefectLib task3 err({liStep}):" + ex.Message + $"||{task.modelType}-{task.thresholds}, {task.thresholdsClass}, {JsonConvert.SerializeObject(task.recAreaThreshold)}"); task.isSucceed = false; task.resultInfo = ex.Message; callback(task); } } } private void run4() { while (IsInit) { if (taskOperationList.Count < 1) { Thread.Sleep(5); continue; } // var task = pop2(); int liStep = 0; try { if (task != null) { Stopwatch stopwatch = Stopwatch.StartNew(); //====运行推理(必需单队列) stopwatch.Start(); IDisposableReadOnlyCollection[] results = yolo3.RunModlel(_onnxSession3, task.tensors); liStep = 1; if(task.modelType == "rj") task.informationList = yolo3.ScreeningResults_YD_RJ(results, task.bmps_resize, task.thresholds, task.thresholdsClass, task.recAreaThreshold); else task.informationList = yolo3.ScreeningResults_YD(results, task.bmps_resize, task.thresholds, task.thresholdsClass, task.recAreaThreshold); liStep = 2; //当前大图上缺陷个数 int currPicDefectCount = 0; for (int x = 0; x < task.informationList.Count(); x++) currPicDefectCount += task.informationList[x].Count(); task.defectCount = currPicDefectCount; stopwatch.Stop(); task.stopwatch[3] = stopwatch.ElapsedMilliseconds; liStep = 3; if (task.informationList.Count > 0) { liStep = 4; //lock (taskTagList) // taskTagList.Add(task); //====打标 ,有缺陷返回缺陷图片数组,没缺陷返回所有图片数组 stopwatch.Restart(); #if OpenSizeDefectComp task.bmps_tag = yolo3.DrawYoloPrediction2Mat(task.bmps_cut, task.informationList, 27, SixLabors.ImageSharp.Color.OrangeRed, task.resize.Width, task.Xmm, task.Ymm, out task.defectInfor2RestorationDesk, out task.defectInfor2BigImage); #else task.bmps_tag = yolo3.DrawYoloPrediction2Mat(task.bmps_cut, task.informationList, 27, SixLabors.ImageSharp.Color.OrangeRed, task.resize.Width, task.Xmm, task.Ymm, out task.defectInfor2RestorationDesk); #endif liStep = 5; //List> SizedefectInfor2RestorationDesk = SysMgr.Instance.GetSizeDefectData(task.Xmm, task.Ymm); //task.defectInfor2RestorationDesk = yolo3.FilteringOverlappingDefects(task.defectInfor2RestorationDesk, SizedefectInfor2RestorationDesk); //liStep = 51; //if (ConfMgr.Instance.SysConfigParams.OpenFlawDistribution) //{ // //大图缺陷坐标转换到图纸坐标 // if (!string.IsNullOrWhiteSpace(task.drawingPagePath) && task.defectInfor2RestorationDesk != null && task.defectInfor2RestorationDesk.Count > 0) // task.defectInfor2RestorationDeskPage = yolo3.DefectDrawGerberImage(task.drawingPagePath, task.defectInfor2RestorationDesk); //} stopwatch.Stop(); task.stopwatch[4] = stopwatch.ElapsedMilliseconds; liStep = 6; task.bmpCompress = yolo3.ResizesMat_4(task.bmp); task.isSucceed = true; callback(task); } else { task.isSucceed = true; callback(task); } } Thread.Sleep(5); } catch (Exception ex) { WarningEvent?.Invoke(WarningEnum.Low, $"DefectLib task4 err({liStep}):" + ex.Message); task.isSucceed = false; task.resultInfo = ex.Message; callback(task); } } } /// /// 已不用 /// private void run5() { while (IsInit) { if (taskTagList.Count < 1) { Thread.Sleep(0); continue; } // var task = pop3(); try { if (task != null) { Stopwatch stopwatch = Stopwatch.StartNew(); //====打标 ,有缺陷返回缺陷图片数组,没缺陷返回所有图片数组 stopwatch.Start(); task.bmps_tag = yolo1.DrawYoloPrediction2Mat(task.bmps_cut, task.informationList, 27, SixLabors.ImageSharp.Color.OrangeRed, task.resize.Width, task.Xmm,task.Ymm,out task.defectInfor2RestorationDesk); stopwatch.Stop(); task.stopwatch[4] = stopwatch.ElapsedMilliseconds; task.isSucceed = true; callback(task); } } catch (Exception ex) { WarningEvent?.Invoke(WarningEnum.Low, "DefectLib task3 err:" + ex.Message); task.isSucceed = false; task.resultInfo = ex.Message; callback(task); } } } private void callback(DefectTask task) { //返回成功/失败,异步调用 if (task.finishEvent != null || (task.finishEvent = finishEvent) != null) //task.finishEvent.BeginInvoke(result, errInfo, res => task.finishEvent.EndInvoke(res), null); System.Threading.ThreadPool.QueueUserWorkItem(waitCallback, task); } //异步回调 WaitCallback waitCallback = new WaitCallback(o => { var task = (DefectTask)o; task.finishEvent(task); }); public class DefectTask { public int stepIndex; public string processName; //2024 03 22 加入模型类型 rj pt; public string modelType; public string drawingPagePath;//图纸路径bmp //图索引 public int index = 0; /// /// 绝对位置-----X2轴 /// public double Xmm; /// /// 绝对位置-----Y轴 /// public double Ymm; /// /// 源文件 /// public Mat bmp; public Mat srcbmp; public Mat bmpCompress;//压缩有缺陷大图 public System.Drawing.Size cut_size = new System.Drawing.Size(592, 532); public System.Drawing.Size resize = new System.Drawing.Size(224, 224); /// /// 阈值 /// public float thresholds = 0.4f; /// /// 种类阈值 /// public string thresholdsClass; /// /// 缺陷在面积内数量 /// public Dictionary recAreaThreshold; /// /// 完成后回调 /// public Action finishEvent; // public long createTime = DateTime.Now.Ticks; public DateTime nowTime; //中间变量,供step2使用 public Mat[] bmps_cut; public Mat[] bmps_resize; //预处理模型 public Tensor[] tensors; //==结果返回 public List[]>> informationList; public int defectCount;//informationList中的缺陷数 public List> defectInfor2RestorationDesk, defectInfor2RestorationDeskPage;//打标缺陷转为图纸的坐标 public Bitmap[] bmps_tag; //打标小图 //bmps_tag.count<=bmps_cut.count bmps_tag.count=informationList.count public bool isSucceed;//转换是否成功 public string resultInfo = "";//成功或失败信息 public long[] stopwatch = new long[5]; public List> defectInfor2BigImage; } public void add(DefectTask task) { lock (taskList) taskList.Add(task); } /// /// 打标 返回每张图的绝对位置X,Y /// /// 缺陷小图索引 /// 拍照位绝对位置-----X2轴 /// 拍照位绝对位置-----Y轴 /// 裁剪小图的宽592 /// 裁剪小图的高532 public List makeTag(int defectBmpIndex, double X, double Y, int imagewidth, int imageheight) { return yolo1.ImageXY(defectBmpIndex, X, Y, imagewidth, imageheight); } /// /// 用相机1查看缺陷点 /// /// /// /// /// /// /// public List viewTag(int defectBmpIndex, double X, double Y, int imagewidth, int imageheight) { return yolo1.ImageXY2(defectBmpIndex, X, Y, imagewidth, imageheight); } private DefectTask pop() { lock (taskList) { if (taskList.Count < 1) return null; //int index = 0;// taskList.FindIndex(p => { return p.isSync; }); //if (index < 0) index = 0; var task = taskList[0]; taskList.RemoveAt(0); return task; } } private DefectTask pop2() { lock (taskOperationList) { if (taskOperationList.Count < 1) return null; //int index = 0;// taskList.FindIndex(p => { return p.isSync; }); //if (index < 0) index = 0; var task = taskOperationList[0]; taskOperationList.RemoveAt(0); return task; } } private DefectTask pop3() { lock (taskTagList) { if (taskTagList.Count < 1) return null; //int index = 0;// taskList.FindIndex(p => { return p.isSync; }); //if (index < 0) index = 0; var task = taskTagList[0]; taskTagList.RemoveAt(0); return task; } } public void Dispose() { stop(); } } }