import 'dart:async'; import 'dart:io'; import 'package:body_detection/models/point3d.dart'; import 'package:body_detection/models/pose_landmark.dart'; import 'package:body_detection/models/pose_landmark_type.dart'; import 'package:rxdart/rxdart.dart'; import 'package:body_detection/body_detection.dart'; import 'package:body_detection/models/image_result.dart'; import 'package:body_detection/models/pose.dart'; import 'package:flutter/material.dart'; import 'package:path_provider/path_provider.dart'; import 'pose_painter.dart'; typedef MeanFilteredData = Iterable>; typedef LandmarkVariations = List>; class PoseDetector extends StatefulWidget { const PoseDetector({Key? key}) : super(key: key); @override State createState() => _PoseDetectorState(); } class _PoseDetectorState extends State { static const buffer = 10; static const _shouldWriteToFile = false; final StreamController _streamController = StreamController.broadcast(); final Directory appDir = Directory('/storage/emulated/0/Android/data/com.example.physigo/files'); Image? _cameraImage; Pose? _detectedPose; Size _imageSize = Size.zero; late Future _startCamera; late Stream _variationsStream; late Stream _meanFilterStream; StreamController _stepExerciseStream = StreamController.broadcast(); @override initState() { super.initState(); _startCamera = _startCameraStream(); _meanFilterStream = _getMeanFilterStream(_streamController.stream); if (_shouldWriteToFile) { _writeDataToFile(_meanFilterStream); } _variationsStream = _meanFilterStream.pairwise().map(_calculateVariations); } LandmarkVariations _calculateVariations(Iterable pairPositions) { final previous = pairPositions.first.toList(); final current = pairPositions.last.toList(); LandmarkVariations variations = []; for (int landmark = 0; landmark < previous.length; landmark++) { final dx = current[landmark][0] - previous[landmark][0]; final dy = current[landmark][1] - previous[landmark][1]; final dz = current[landmark][2] - previous[landmark][2]; variations.add([dx.roundToDouble(), dy.roundToDouble(), dz.roundToDouble()]); } return variations; } void _writeDataToFile(Stream stream) { File meanFilteredData = File("${appDir.path}/meanFilteredData.csv"); if (meanFilteredData.existsSync()) meanFilteredData.deleteSync(); stream.listen((meanPositions) { for (var position in meanPositions) { final str = "${position[0]}, ${position[1]}, ${position[2]};"; meanFilteredData.writeAsStringSync(str, mode: FileMode.append); } meanFilteredData.writeAsStringSync("\n", mode: FileMode.append); }); } Stream _getMeanFilterStream(Stream stream) { return stream .where((pose) => pose.landmarks.isNotEmpty) .map((pose) => pose.landmarks.where((landmark) => authorizedType.contains(landmark.type)).toList()) // Get last [buffer] poses .bufferCount(buffer, 1) // Swap matrix [buffer] * [authorizedType.length] .map(_swapMatrixDimensions) // For every landmarks, get meanFilter of size [buffer] .map((filteredLandmarks) => filteredLandmarks.map(_meanFilter)); } List _meanFilter(List landmarks) { return landmarks .map((landmark) => landmark.position) .map((position) => [ position.x / buffer, position.y / buffer, position.z / buffer, ]) .reduce((value, element) => [ value[0] + element[0], value[1] + element[1], value[2] + element[2], ]); } List> _swapMatrixDimensions(List> matrix) { final height = matrix.length; final width = matrix[0].length; List> newMatrix = []; for (int col = 0; col < width; col++) { List newRow = []; for (int row = 0; row < height; row++) { newRow.add(matrix[row][col]); } newMatrix.add(newRow); } return newMatrix; } Future _startCameraStream() async { await BodyDetection.startCameraStream(onFrameAvailable: _handleCameraImage, onPoseAvailable: _handlePose); await BodyDetection.enablePoseDetection(); } Future _stopCameraStream() async { await BodyDetection.disablePoseDetection(); await BodyDetection.stopCameraStream(); } void _handleCameraImage(ImageResult result) { if (!mounted) return; // To avoid a memory leak issue. // https://github.com/flutter/flutter/issues/60160 PaintingBinding.instance?.imageCache?.clear(); PaintingBinding.instance?.imageCache?.clearLiveImages(); final image = Image.memory( result.bytes, gaplessPlayback: true, fit: BoxFit.contain, ); setState(() { _cameraImage = image; _imageSize = result.size; }); } void _handlePose(Pose? pose) { if (!mounted) return; if (pose != null) _streamController.add(pose); setState(() { _detectedPose = pose; }); } @override void dispose() { _stopCameraStream(); _streamController.close(); super.dispose(); } @override Widget build(BuildContext context) { return FutureBuilder( future: _startCamera, builder: (context, snapshot) { if (snapshot.connectionState == ConnectionState.waiting) { return const Center(child: CircularProgressIndicator()); } return Column( children: [ Center( child: CustomPaint( // size: _imageSize, child: _cameraImage, foregroundPainter: PosePainter( pose: _detectedPose, imageSize: _imageSize, ), ), ), StreamBuilder( stream: _meanFilterStream, builder: (context, snapshot) { if (!snapshot.hasData) { return CircularProgressIndicator(); } final landmarks = snapshot.data!.toList(); final xRightHip = landmarks[8][0]; final xRightKnee = landmarks[10][0]; final xDistanceHipKnee = (xRightHip - xRightKnee).abs(); final yRightHip = landmarks[8][1]; final yRightKnee = landmarks[10][1]; final yDistanceHipKnee = (yRightHip - yRightKnee).abs(); var message = "IN BETWEEN"; if (xDistanceHipKnee < 30) { message = "START"; _stepExerciseStream.add(Colors.green); } else if (yDistanceHipKnee < 40) { message = "END"; _stepExerciseStream.add(Colors.red); } else { _stepExerciseStream.add(Colors.yellow); } final zRightHip = landmarks[8][2]; final zRightKnee = landmarks[10][2]; final zDistanceHipKnee = (zRightHip - zRightKnee).abs(); return Text("$zDistanceHipKnee", style: TextStyle(fontSize: 40)); }, ), StreamBuilder( stream: _stepExerciseStream.stream, builder: (context, snapshot) { if (!snapshot.hasData) { return CircularProgressIndicator(); } return Container( height: 100, width: 100, color: snapshot.data!, ); }, ), ], ); }, ); } static const authorizedType = [ PoseLandmarkType.nose, PoseLandmarkType.leftShoulder, PoseLandmarkType.rightShoulder, PoseLandmarkType.leftElbow, PoseLandmarkType.rightElbow, PoseLandmarkType.leftWrist, PoseLandmarkType.rightWrist, PoseLandmarkType.leftHip, PoseLandmarkType.rightHip, PoseLandmarkType.leftKnee, PoseLandmarkType.rightKnee, PoseLandmarkType.leftAnkle, PoseLandmarkType.rightAnkle, ]; } /* GETTING IN POSITION: CHECK IF EVERY NECESSARY JOINT ARE ON SCREEN (reliability > 0.8) CHECK IF START POSITION IS OKAY (for squat, if knee and hip on same x coordinate) COUTING REPETITION: FROM BEGINNING TO END: - BEGINNING: defined by start position, get position of interesting joint - END: defined by positions/distance interesting joints (knee and hip same level for squat, elbow and should same level for push up) */