import 'dart:async'; import 'dart:io'; import 'package:body_detection/models/point3d.dart'; import 'package:body_detection/models/pose_landmark.dart'; import 'package:body_detection/models/pose_landmark_type.dart'; import 'package:rxdart/rxdart.dart'; import 'package:body_detection/body_detection.dart'; import 'package:body_detection/models/image_result.dart'; import 'package:body_detection/models/pose.dart'; import 'package:flutter/material.dart'; import 'package:path_provider/path_provider.dart'; import 'pose_painter.dart'; class PoseDetector extends StatefulWidget { const PoseDetector({Key? key}) : super(key: key); @override State createState() => _PoseDetectorState(); } class _PoseDetectorState extends State { Image? _cameraImage; Pose? _detectedPose; Size _imageSize = Size.zero; late Future _startCamera; final StreamController _streamController = StreamController.broadcast(); final Directory appDir = Directory('/storage/emulated/0/Android/data/com.example.physigo/files'); @override initState() { super.initState(); _startCamera = _startCameraStream(); _writeDataToFile(); } void _writeDataToFile() { const buffer = 10; File meanFilteredData = File("${appDir.path}/meanFilteredData.csv"); if (meanFilteredData.existsSync()) meanFilteredData.deleteSync(); // TODO: get positions variation // TODO: detect excentric/concentric part of the movement (derivative negative or positive) _streamController.stream .where((pose) => pose.landmarks.isNotEmpty) .map((pose) => pose.landmarks.where((landmark) => authorizedType.contains(landmark.type)).toList()) .bufferCount(buffer, 1) .listen((filteredLandmarks) { // inverse height and width of the matrix List> bufferedPositions = []; for (int j = 0; j < filteredLandmarks[0].length; j++) { List positions = []; for (int i = 0; i < filteredLandmarks.length; i++) { positions.add(filteredLandmarks[i][j]); } bufferedPositions.add(positions); } // mean filters of the buffer points final meanPositions = bufferedPositions.map((landmarks) { return landmarks .map((landmark) => landmark.position) .map((position) => [ position.x / buffer, position.y / buffer, position.z / buffer, ]) .reduce((value, element) => [ value[0] + element[0], value[1] + element[1], value[2] + element[2], ]); }); for (var position in meanPositions) { final str = "${position[0]}, ${position[1]}, ${position[2]};"; meanFilteredData.writeAsStringSync(str, mode: FileMode.append); } meanFilteredData.writeAsStringSync("\n", mode: FileMode.append); }); } Future _startCameraStream() async { await BodyDetection.startCameraStream(onFrameAvailable: _handleCameraImage, onPoseAvailable: _handlePose); await BodyDetection.enablePoseDetection(); } Future _stopCameraStream() async { await BodyDetection.disablePoseDetection(); await BodyDetection.stopCameraStream(); } void _handleCameraImage(ImageResult result) { if (!mounted) return; // To avoid a memory leak issue. // https://github.com/flutter/flutter/issues/60160 PaintingBinding.instance?.imageCache?.clear(); PaintingBinding.instance?.imageCache?.clearLiveImages(); final image = Image.memory( result.bytes, gaplessPlayback: true, fit: BoxFit.contain, ); setState(() { _cameraImage = image; _imageSize = result.size; }); } void _handlePose(Pose? pose) { if (!mounted) return; if (pose != null) _streamController.add(pose); setState(() { _detectedPose = pose; }); } @override void dispose() { _stopCameraStream(); _streamController.close(); super.dispose(); } @override Widget build(BuildContext context) { return FutureBuilder( future: _startCamera, builder: (context, snapshot) { if (snapshot.connectionState == ConnectionState.waiting) { return const Center(child: CircularProgressIndicator()); } return Center( child: CustomPaint( size: _imageSize, child: _cameraImage, foregroundPainter: PosePainter( pose: _detectedPose, imageSize: _imageSize, ), ), ); }, ); } static const authorizedType = [ PoseLandmarkType.nose, PoseLandmarkType.leftShoulder, PoseLandmarkType.rightShoulder, PoseLandmarkType.leftElbow, PoseLandmarkType.rightElbow, PoseLandmarkType.leftWrist, PoseLandmarkType.rightWrist, PoseLandmarkType.leftHip, PoseLandmarkType.rightHip, PoseLandmarkType.leftKnee, PoseLandmarkType.rightKnee, PoseLandmarkType.leftAnkle, PoseLandmarkType.rightAnkle, ]; }