EmguCV学习笔记 VB.Net 11.6 图像分割

 版权声明：本文为博主原创文章，转载请在显著位置标明本文出处以及作者网名，未经作者允许不得用于商业目的。

EmguCV是一个基于OpenCV的开源免费的跨平台计算机视觉库,它向C#和VB.NET开发者提供了OpenCV库的大部分功能。

教程VB.net版本请访问：EmguCV学习笔记 VB.Net 目录-CSDN博客

教程C#版本请访问：EmguCV学习笔记 C# 目录-CSDN博客

笔者的博客网址：https://blog.csdn.net/uruseibest

教程配套文件及相关说明以及如何获得pdf教程和代码，请移步：EmguCV学习笔记

学习VB.Net知识，请移步： vb.net 教程 目录_vb中如何用datagridview-CSDN博客

 学习C#知识，请移步：C# 教程 目录_c#教程目录-CSDN博客

11.6 图像分割

11.6.1 语义分割Fcn

FCN（Fully Convolutional Network）是一种流行的语义分割算法，它通过将传统的卷积神经网络转化为全卷积网络来实现像素级别的语义分割，将输入图像传入网络中，得到每个像素的标签，并根据标签来分割出不同的物体。FCN在语义分割任务中表现优秀，可以实现像素级别的精细分割，被广泛应用于自动驾驶、智能监控等领域。

【代码位置：frmChapter11】Button5_Click、getColorTable、showColorTable

    '语义分割fcn

    Private Sub Button5_Click(sender As Object, e As EventArgs) Handles Button5.Click

        '对象分类，object_detection_classes_pascal_voc.txt文件提供了21类对象（含background）

        Dim classnames() As String

        classnames = File.ReadAllLines("C:\learnEmgucv\fcn\object_detection_classes_pascal_voc.txt")

        '获得输出颜色表

        Dim colorTable() As Bgr = getColorTable()

        '显示颜色表及对应对象名称

        Call showColorTable(classnames, colorTable)

        '需要测试的图像文件

        Dim m As New Mat("C:\learnEmgucv\dnntest1.jpg", ImreadModes.Color)

        Dim hm As Single = m.Height

        Dim wm As Single = m.Width

        Dim net As Dnn.Net

        net = DnnInvoke.ReadNetFromCaffe("C:\learnEmgucv\fcn\fcn8s-heavy-pascal.prototxt",

                                         "C:\learnEmgucv\fcn\fcn8s-heavy-pascal.caffemodel"

                                        )

        Dim mcopy As New Mat

        CvInvoke.Resize(m, mcopy, New Drawing.Size(500, 500))

        Dim blob As Mat

        blob = DnnInvoke.BlobFromImage(mcopy, 1, New Size(500, 500),

                                       New MCvScalar(0， 0， 0), False, False)

        net.SetInput(blob)

        '

        Dim mout As New Mat

        mout = net.Forward()

        Dim fout(,,,) As Single

        fout = mout.GetData()

        '通道数=21，即21类对象

        Dim chan As Integer = fout.GetLength(1)

        '高度=500

        Dim row As Integer = fout.GetLength(2)

        '宽度=500

        Dim col As Integer = fout.GetLength(3)

        '记录21个通道对应坐标点的最大值

        Dim matrMaxValue As New Matrix(Of Single)(New Size(row, col))

        matrMaxValue.SetZero()

        ''记录对应通道号

        'Dim matrMaxChan As New Matrix(Of Single)(New Size(row, col))

        'matrMaxChan.SetZero()

        '记录对应颜色

        Dim imgOut As New Image(Of Bgr, Byte)(col, row)

        imgOut.SetZero()

        '遍历21个通道

        For c As Integer = 0 To chan - 1

            For h As Integer = 0 To row - 1

                '遍历高度和宽度，获得对应坐标的值

                For w As Integer = 0 To col - 1

                    '比较最大值

                    If fout(0, c, h, w) > matrMaxValue(h, w) Then

                        '取得最大值

                        matrMaxValue(h, w) = fout(0, c, h, w)

                        ''取得通道号

                        'matrMaxChan(h, w) = c

                        '最重要的是获取通道对应颜色表的值

                        imgOut(h, w) = colorTable(c)

                    End If

                Next

            Next

        Next

        '显示输出的图像

        'ImageBox1.Image = imgOut.Mat

        '设置掩膜

        Dim mask As New Mat

        mask = imgOut.Mat

        '掩膜大小必须和源图像一致

        CvInvoke.Resize(mask, mask, m.Size)

        '将上面的输出图像和源图像叠加

        Dim mFinalOut As New Mat

        CvInvoke.AddWeighted(m, 0.3, mask, 0.7, 0, mFinalOut)

        ImageBox1.Image = mFinalOut

    End Sub

    '为了更好地观察，这里没有使用随机颜色

    Private Function getColorTable() As Bgr()

        Dim newColors(20) As Bgr

        newColors(0) = New Bgr(0, 0, 0)        'background

        newColors(1) = New Bgr(128, 0, 0)      'aeroplane

        newColors(2) = New Bgr(0, 128, 0)      'bicycle

        newColors(3) = New Bgr(128, 128, 0)    'bird

        newColors(4) = New Bgr(0, 0, 128)      'boat

        newColors(5) = New Bgr(128, 0, 128)    'bottle

        newColors(6) = New Bgr(0, 128, 128)    'bus

        newColors(7) = New Bgr(128, 128, 128)  'car

        newColors(8) = New Bgr(255, 0, 0)      'cat

        newColors(9) = New Bgr(0, 255, 0)      'chair

        newColors(10) = New Bgr(0, 0, 255)     'cow

        newColors(11) = New Bgr(255, 255, 0)   'diningtable

        newColors(12) = New Bgr(64, 0, 128)    'dog

        newColors(13) = New Bgr(192, 0, 128)   'horse

        newColors(14) = New Bgr(64, 128, 128)  'motorbike

        newColors(15) = New Bgr(192, 128, 128) 'person

        newColors(16) = New Bgr(0, 64, 0)      'pottedplant

        newColors(17) = New Bgr(128, 64, 64)   'sheep

        newColors(18) = New Bgr(0, 192, 0)     'sofa

        newColors(19) = New Bgr(128, 192, 0)   'train

        newColors(20) = New Bgr(0, 64, 128)    'tvmonitor

        Return newColors

    End Function

    '显示颜色表及对象名称

    '参数1：对象名称的字符串数组

    '参数2：bgr颜色数组

    Private Sub showColorTable(ByVal names() As String, ByVal colors() As Bgr)

        Dim lstmoutV As New List(Of Mat)

        For i As Integer = 0 To 20

            Dim moutV As New Mat(40, 200, DepthType.Cv8U, 3)

            moutV.SetTo(New MCvScalar(colors(i).Blue, colors(i).Green, colors(i).Red))

            CvInvoke.PutText(moutV, names(i), New Point(20, 25), FontFace.HersheyTriplex, 0.4, New MCvScalar(255, 255, 255))

            lstmoutV.Add(moutV)

        Next

        Dim mout As New Mat

        '垂直方向拼接

        CvInvoke.VConcat(lstmoutV.ToArray, mout)

        CvInvoke.Imshow("colortable", mout)

End Sub

输出结果如下图所示：

 

图11-4 FCN分割获得不同对象区域

11.6.2 实例分割 MASK RCNN

Mask RCNN是一种基于Faster R-CNN的实例分割算法，它是一种联合目标检测和语义分割的方法，能够同时检测图像中的对象并得到每个对象的位置、类别和像素级别的分割结果。Mask RCNN在实例分割任务中表现优秀，是目前最先进的实例分割算法之一，被广泛应用于自动驾驶、智能监控等领域。

【代码位置：frmChapter11】Button6_Click、getRadomColor

   '实例分割 mask rcnn

    Private Sub Button6_Click(sender As Object, e As EventArgs) Handles Button6.Click

        '对象分类，object_detection_classes_coco.txt文件提供了90类对象（含background）

        Dim classnames() As String

        classnames = File.ReadAllLines("C:\learnEmgucv\maskrcnn\object_detection_classes_coco.txt")

        '需要测试的图像文件

        Dim m As New Mat("C:\learnEmgucv\dnntest.jpg", ImreadModes.Color)

        Dim hm As Single = m.Height

        Dim wm As Single = m.Width

        Dim net As Dnn.Net

        net = DnnInvoke.ReadNetFromTensorflow("C:\learnEmgucv\maskrcnn\frozen_inference_graph.pb",

                             "C:\learnEmgucv\maskrcnn\mask_rcnn_inception_v2_coco_2018_01_28.pbtxt"

                              )

        Dim mcopy As New Mat

        mcopy = m.Clone

        Dim blob As Mat

        blob = DnnInvoke.BlobFromImage(mcopy, 1, New Size(500, 500),

                                       New MCvScalar(0， 0， 0), False, False)

        net.SetInput(blob)

        Dim names(1) As String

        names（0） = "detection_out_final"

        names（1） = "detection_masks"

        '返回的mout包含两个mat

        Dim mout As New VectorOfMat

        net.Forward(mout, names)

        '第一个mat标识返回的置信度候选框，是一个四维数组

        Dim moutBox As New Mat

        moutBox = mout(0)

        '返回维度：

        '第1维：1，

        '第2维：1，

        '第3维：100，100个候选置信矩形框

        '第4维：7，0：?;1:对应类别；2：置信度；3-6：候选框位置（源图像百分比）

        Dim foutBox(,,,) As Single

        foutBox = moutBox.GetData()

        '第二个mat标识返回的掩膜，是一个四维数组   

        Dim moutMask As New Mat

        moutMask = mout(1)

        '返回维度：

        '第1维：100，100个对象对应的100个掩膜

        '第2维：90，对象的置信度

        '第3维：15，掩膜高度

        '第4维：15，掩膜宽度

        Dim foutMask(,,,) As Single

        foutMask = moutMask.GetData()

        Dim maskH As Integer = 15   'foutMask.GetLength(2)

        Dim maskW As Integer = 15   'foutMask.GetLength(3)

        '新建Mat，用来在这上面绘制掩膜

        Dim mbg As New Mat(New Size(m.Width, m.Height), DepthType.Cv8U, 3)

        mbg.SetTo(New MCvScalar(0, 0, 0))

        For i As Integer = 0 To 100 - 1

            '置信度

            Dim conf As Single = foutBox(0, 0, i, 2)

            '当置信度满足时

            If conf > 0.53 Then

                '对应检测对象的序号

                Dim objID As Integer = foutBox(0, 0, i, 1)

                Dim ltX As Single = foutBox(0, 0, i, 3) * wm '左上角X

                Dim ltY As Single = foutBox(0, 0, i, 4) * hm '左上角Y

                Dim rbX As Single = foutBox(0, 0, i, 5) * wm '右下角X

                Dim rbY As Single = foutBox(0, 0, i, 6) * hm '右下角Y

                Dim w As Single = rbX - ltX '宽度

                Dim h As Single = rbY - ltY '高度

                '绘制包围矩形框

                CvInvoke.Rectangle(m, New Rectangle(ltX, ltY, w, h), New MCvScalar(0, 0, 255), 1)

                '绘制对象名称

                CvInvoke.PutText(m, classnames(objID), New Point(ltX, ltY - 10), FontFace.HersheyTriplex, 0.3, New MCvScalar(255, 0, 0))

                '开始处理掩膜

                Dim mmask As New Mat()

                '掩膜大小为15*15，对应foutMask最后两个维度

                Dim bmask(14, 14) As Single

                For j As Integer = 0 To 14

                    For k As Integer = 0 To 14

                        bmask(j, k) = foutMask(i, objID, j, k)

                    Next

                Next

                '将数组转为Mat

                Dim matrmask As New Matrix(Of Single)(bmask)

                mmask = matrmask.Mat

                '大小放大与对应包围矩形框一致

                CvInvoke.Resize(mmask, mmask, New Drawing.Size(w, h))

                '二值化

                CvInvoke.Threshold(mmask, mmask, 0.3, 255, ThresholdType.Binary)

                '由于本身是CV32F，需要处理为CV8U，才能使用FindContours

                mmask.ConvertTo(mmask, DepthType.Cv8U)

                '定义关注区域，当修改关注区域时，也就修改了源图像

                Dim mRoi As New Mat(mbg, New Rectangle(ltX, ltY, w, h))

                '查找轮廓

                Dim contours As New VectorOfVectorOfPoint

                CvInvoke.FindContours(mmask, contours, Nothing, RetrType.External, ChainApproxMethod.ChainApproxSimple)

                '填充轮廓

                CvInvoke.DrawContours(mRoi, contours, -1, getRadomColor(), -1)

            End If

        Next

        Dim mresult As New Mat

        '加法

        CvInvoke.AddWeighted(m, 1, mbg, 0.4, 0, mresult)

        ImageBox1.Image = mresult

    End Sub

    '获得随机颜色

    Private Function getRadomColor() As MCvScalar

        Dim rd As New Random(Now.Millisecond)

        Dim r, g, b As Byte

        b = rd.Next(256)

        g = rd.Next(256)

        r = rd.Next(256)

        Return New MCvScalar(b, g, r)

End Function

输出结果如下图所示：

 

图11-5 Mask RCNN 分割得到对象区域和类别