【AI】利用简单神经网络做动作识别——基于coco关键点(ai知识图谱开发)


本文主要分享【ai知识图谱开发】,技术文章【【AI】利用简单神经网络做动作识别——基于coco关键点】为【Dreamcatcher风】投稿,如果你遇到机器学习/深度学习相关问题,本文相关知识或能到你。

ai知识图谱开发

前言

         coco数据集包含了关键点检测,如果想利用提取到的这些关键点做行为识别,该怎么办呢?下文主要通过搭建一个简单神经网络(多层感知机)来做关键点的分类。

        任务:假如需要对打电话和玩手机的动作做分类。

开始

        第一步,需要利用HRNet提取图像中人物的关键点,所需工程代码详见该博文:

YOLOv5+姿态估计HRnet与SimDR检测视频中的人体关键点_围白的尾巴的博客-CSDN博客_yolov5人体检测

         第二步,需要重写main.py文件里的代码:

import argparse
import imp
import time
import os
import cv2 as cv
import numpy as np
from pathlib import Path
from Point_detect import Points
from lib.utils.visualization import draw_points_and_skeleton,joints_dict
import csv
from tqdm import tqdm
 
def image_detect(opt):

    skeleton = joints_dict()['coco']['skeleton']
    hrnet_model = Points(model_name='hrnet', opt=opt,resolution=(384,288))  
    
    pic_file = os.listdir(opt.source)
    # print(pic_file,'\n')
    for pic in tqdm(pic_file):
        img0 = cv.imread(os.path.join(opt.source, pic))
        frame = img0.copy()
    #predict
        pred, bbox = hrnet_model.predict(img0) # 这里修改了Point_detect.py文件, 多获取一个bbox坐标信息

        ### 转为一位数组保存, 后面再处理(这里涉及到多个人的情况)
        for target in range(len(pred)):
            pred_flatten = pred[target].ravel()[0:33] # 只获取前11个关键点
            new = []
            tag = 0
            for i in range(len(pred_flatten)):
                if tag != 2:
                    new.append(pred_flatten[i])
                    tag += 1
                else:
                    tag = 0

            point_num = 1

            base_x, base_y = pred_flatten[0], pred_flatten[1]  # 以这个为基准点(鼻子)

            w , h = img0.shape[1]-base_x, img0.shape[0]-base_y
            k = 0
            with open('keypoint_call.csv', 'a', encoding='utf8') as name:
                for i in pred_flatten:
                    if i > 1: # 大于1的部分才是坐标, 小于1的是置信度
                        # 归一化坐标, 计算关键点相对图片上的位置
                        # 注意: 要考虑在左右两边的情况, 采用对称计算方法
                        if i-base_x < 0:
                            x_ = base_x+(base_x-i)
                            x = -(x_-base_x)/w
                        else:
                            x = (i-base_x)/w
                        if i-base_y < 0:
                            y_ = base_y+(base_y-i)
                            y = -(y_-base_y)/w
                        else:
                            y = (i-base_y)/h

                        if k % 2 == 0: # 奇数时写入x, 偶数时写入y
                            name.write(str(x)+',') if point_num < len(pred_flatten) else name.write(str(x)+'\n')
                        else:
                            name.write(str(y)+',') if point_num < len(pred_flatten) else name.write(str(y)+'\n')
                    else:
                        name.write(str(i)+',') if point_num < len(pred_flatten) else name.write(str(i)+'\n')
                    point_num += 1
                    k += 1

        # 可视化和保存
        # for i, pt in enumerate(pred):
        #     frame = draw_points_and_skeleton(frame, pt, skeleton)
        #     name = 'test_result'+pic+'.jpg'
        #     cv.imwrite(os.path.join('D:/save',name), frame)

        ### 注意, pt就是关键点, 是一个len=17的tuple, 每一个元素对应一个点

 
def video_detect(opt):
    hrnet_model = Points(model_name='hrnet', opt=opt, resolution=(384, 288))  # resolution = (384,288)  or (256,192)

    skeleton = joints_dict()['coco']['skeleton']
 
    cap = cv.VideoCapture(opt.source)
    if opt.save_video:
        fourcc = cv.VideoWriter_fourcc(*'MJPG')
        out = cv.VideoWriter('data/runs/{}_out.avi'.format(os.path.basename(opt.source).split('.')[0]), fourcc, 24, (int(cap.get(3)), int(cap.get(4))))
    while cap.isOpened():
        ret, frame = cap.read()
        if not ret:
            break
        pred = hrnet_model.predict(frame)

        for pt in pred:
            frame = draw_points_and_skeleton(frame,pt,skeleton)
        if opt.show:
            cv.imshow('result', frame)
        if opt.save_video:
            out.write(frame)
        if cv.waitKey(1) == 27:
            break
    out.release()
    cap.release()
    cv.destroyAllWindows()
# video_detect(0)
if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--source', type=str, default='D:/call', help='source')  # 存放图片的文件夹
    parser.add_argument('--detect_weight', type=str, default="./yolov5/weights/yolov5x.pt", help='e.g "./yolov5/weights/yolov5x.pt"')
    parser.add_argument('--save_video', action='store_true', default=False,help='save results to *.avi')
    parser.add_argument('--show', action='store_true', default=True, help='save results to *.avi')
    parser.add_argument('--device', default='cpu', help='cuda device, i.e. 0 or 0,1,2,3 or cpu')
    parser.add_argument('--conf-thres', type=float, default=0.25, help='object confidence threshold')
    parser.add_argument('--iou-thres', type=float, default=0.45, help='IOU threshold for NMS')
    parser.add_argument('--classes', nargs='+', type=int, help='filter by class: --class 0, or --class 0 2 3')
    parser.add_argument('--agnostic-nms', action='store_true', help='class-agnostic NMS')
    parser.add_argument('--augment', action='store_true', help='augmented inference')
 
    opt = parser.parse_args()
    image_detect(opt)

        注意,这里有一个地方很重要,就是如何对关键点做归一化处理

        由于每张图片中目标所占图片大小和所在位置完全不同,因此不能直接以图片左上角作为原点去寻找关键点位置,需要利用某个关键点作为原点我使用的方法是借鉴YOLO v5对Bounding Box做归一化的方式,核心思想是(x/w , y/h),但需要做一些修改:

         获得一个csv文件:

         第三步,搭建多层感知机,训练分类模型:

import csv
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split

from keras import models, optimizers
from keras.models import Sequential
from keras.layers import Dense, Dropout, BatchNormalization
import keras.backend as K
from keras.callbacks import LearningRateScheduler
from keras.utils.np_utils import *

### 读取文件
data_call = pd.read_csv('D:/keypoint_call.csv') # 这是打电话关键点的文件
data_play = pd.read_csv('D:/keypoint_play.csv') # 这是玩手机关键点的文件
data_no = pd.read_csv('D:/keypoint_no.csv') # 这是负样本文件

### 正负样本拼接, 归一化
train = pd.DataFrame(pd.concat([data_play, data_call], ignore_index=True))
### 补齐17个关键点
for i in range(len(train.columns), 34):
    train.insert(loc=i, column=str(i+1), value=0)
### 独热编码
target = [1 if num < len(data_play) else 2 for num in range(len(train))] # 0:负样本  1:玩手机  2:打电话
train = pd.DataFrame(pd.concat([train, data_no], ignore_index=True))
for i in range(len(data_no)):
    target.append(0)
target = np.array(target)
target = to_categorical(target, 3)

### 分配训练集和验证集
x_train, x_val, y_train, y_val = train_test_split(train, target, test_size=0.2, random_state=2022)

### 学习率调整方案
def scheduler(epoch):
    if epoch % 30 == 0 and epoch != 0:
        lr = K.get_value(model.optimizer.lr)
        K.set_value(model.optimizer.lr, lr * 0.1)
        print("lr changed to {}".format(lr * 0.1))
    return K.get_value(model.optimizer.lr)

# 训练
seed = 9
np.random.seed(seed)

model = Sequential()
model.add(Dense(256, input_dim=34, activation='relu'))
# model.add(Dropout(0.5)),
model.add(Dense(128, input_dim=256, activation='relu'))
# model.add(Dropout(0.5)), 
# model.add(BatchNormalization()),
model.add(Dense(64, input_dim=128, activation='relu')),
# model.add(Dropout(0.15)),
model.add(Dense(3, activation='softmax'))

model.compile(optimizer=optimizers.adam_v2.Adam(lr=0.001), # 学习率很重要!
              loss='categorical_crossentropy',
              metrics=['accuracy'])

reduce_lr = LearningRateScheduler(scheduler)
history = model.fit(np.array(x_train),
                    np.array(y_train),
                    epochs=100,
                    batch_size=64, # bs很重要!
                    validation_data=(np.array(x_val), np.array(y_val)),
                    callbacks=[reduce_lr]
                    )
### 可视化训练结果
acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']

epochs = range(1, len(acc) + 1)

plt.figure(figsize=(6,6))
plt.plot(epochs, acc, 'b', label='Train acc',color='lightseagreen')
plt.plot(epochs, val_acc, 'b', label='Val acc',color='tomato')
plt.xlabel('Epochs')
plt.ylabel('acc')
plt.legend()
plt.show()

### 保存模型
model.save('D:/keypoint_model.h5')

 

 

         第四步,推理:

        对main.py稍作修改即可:(可以用一个if来控制生成csv还是推理,可自行修改)

import argparse
import imp
from random import randrange
import time
import os
import cv2 as cv
import numpy as np
from pathlib import Path
from Point_detect import Points
from lib.utils.visualization import draw_points_and_skeleton,joints_dict
import csv
from tqdm import tqdm
from keras.models import load_model
 
def image_detect(opt):

    skeleton = joints_dict()['coco']['skeleton']
    hrnet_model = Points(model_name='hrnet', opt=opt,resolution=(384,288))  

    keypoint_model = load_model('keypoint_model.h5')
    
    pic_file = os.listdir(opt.source)
    # print(pic_file,'\n')
    for pic in pic_file:
        print('pic = {} :'.format(pic))
        img0 = cv.imread(os.path.join(opt.source, pic))
        frame = img0.copy()                                 
        pred, bbox = hrnet_model.predict(img0)

        ### 转为一位数组保存, 后面再处理(这里涉及到多个人的情况)
        for target in range(len(pred)):
            pred_flatten = pred[target].ravel()[0:33] # 只获取前11个关键点
            point = []
            tag = 0
            for i in range(len(pred_flatten)):
                if tag != 2:
                    point.append(pred_flatten[i])
                    tag += 1
                else:
                    tag = 0

            base_x, base_y = point[0], point[1]  # 以这个为基准点(鼻子)

            w , h = img0.shape[1]-base_x, img0.shape[0]-base_y

            xy = []
            k = 0
            for i in point:
                if i-base_x < 0:
                    x_ = base_x+(base_x-i)
                    x = -(x_-base_x)/w
                else:
                    x = (i-base_x)/w
                if i-base_y < 0:
                    y_ = base_y+(base_y-i)
                    y = -(y_-base_y)/w
                else:
                    y = (i-base_y)/h

                if k % 2 == 0:
                    xy.append(x)
                else:
                    xy.append(y)
                k += 1

            for i in range(12):
                xy.append(0)

            out = keypoint_model.predict(np.array(xy).reshape(1,-1))
            print('out = {}'.format(out))
            predict = np.argmax(out)
            tag = 'play phone' if predict == 1 else ('call' if predict == 2 else 'normal')
            print('tag = {}\n'.format(tag))  

            # 保存bbox和tag
            if tag == 'call':
                cv.rectangle(frame, (bbox[target][0],bbox[target][1]), (bbox[target][2],bbox[target][3]), (255,0,0), thickness=2)
                cv.putText(frame, tag, (bbox[target][0],bbox[target][1]-10), cv.FONT_HERSHEY_SIMPLEX, color=(255,0,0), fontScale = 0.75, thickness=2)
            else:
                cv.rectangle(frame, (bbox[target][0],bbox[target][1]), (bbox[target][2],bbox[target][3]), (0,0,255), thickness=2)
                cv.putText(frame, tag, (bbox[target][0],bbox[target][1]-10), cv.FONT_HERSHEY_SIMPLEX, color=(0,0,255), fontScale = 0.75, thickness=2)
        
        name = 'test_result'+pic+'.jpg'
        cv.imwrite(os.path.join('D:/save',name), frame)
 
def video_detect(opt):
    hrnet_model = Points(model_name='hrnet', opt=opt, resolution=(384, 288))  # resolution = (384,288)  or (256,192)
    skeleton = joints_dict()['coco']['skeleton']
 
    cap = cv.VideoCapture(opt.source)
    if opt.save_video:
        fourcc = cv.VideoWriter_fourcc(*'MJPG')
        out = cv.VideoWriter('data/runs/{}_out.avi'.format(os.path.basename(opt.source).split('.')[0]), fourcc, 24, (int(cap.get(3)), int(cap.get(4))))
    while cap.isOpened():
        ret, frame = cap.read()
        if not ret:
            break
        pred = hrnet_model.predict(frame)
        for pt in pred:
            frame = draw_points_and_skeleton(frame,pt,skeleton)
        if opt.show:
            cv.imshow('result', frame)
        if opt.save_video:
            out.write(frame)
        if cv.waitKey(1) == 27:
            break
    out.release()
    cap.release()
    cv.destroyAllWindows()
# video_detect(0)
if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--source', type=str, default='D:/aa', help='source')  # 存放图片的文件夹
    parser.add_argument('--detect_weight', type=str, default="./yolov5/weights/yolov5x.pt", help='e.g "./yolov5/weights/yolov5x.pt"')
    parser.add_argument('--save_video', action='store_true', default=False,help='save results to *.avi')
    parser.add_argument('--show', action='store_true', default=True, help='save results to *.avi')
    parser.add_argument('--device', default='cpu', help='cuda device, i.e. 0 or 0,1,2,3 or cpu')
    parser.add_argument('--conf-thres', type=float, default=0.25, help='object confidence threshold')
    parser.add_argument('--iou-thres', type=float, default=0.45, help='IOU threshold for NMS')
    parser.add_argument('--classes', nargs='+', type=int, help='filter by class: --class 0, or --class 0 2 3')
    parser.add_argument('--agnostic-nms', action='store_true', help='class-agnostic NMS')
    parser.add_argument('--augment', action='store_true', help='augmented inference')
 
    opt = parser.parse_args()
    image_detect(opt)

 最后,看看结果

注1:当时没加负样本,所以效果仅供娱乐 :)

注2:此处露个相,戴帽子那位是本人(一气之下把头发剪太短,在恢复ing)

 

 

后记:整个过程比较简单,遇到的一个小坑,主要还是在归一化那里。之前没考虑到这个问题,在实测中发现模型完全胡乱判断,最后才找到原因,解决它。


如有新的想法,期待交流探讨


 关注我的微信公众号“风的思考笔记”,我们一起思考当下,探索未来自由之路。

图片

本文《【AI】利用简单神经网络做动作识别——基于coco关键点》版权归Dreamcatcher风所有,引用【AI】利用简单神经网络做动作识别——基于coco关键点需遵循CC 4.0 BY-SA版权协议。


注意!

本站转载的文章为个人学习借鉴使用,本站对版权不负任何法律责任。如果侵犯了您的隐私权益,请联系我们删除。



 
  © 2014-2022 ITdaan.com 联系我们: