to reset

app/lib/exercises/exercises_validation/widgets/pose_detector.dart

@@ -13,6 +13,9 @@ import 'package:path_provider/path_provider.dart';
 
 import 'pose_painter.dart';
 
+typedef MeanFilteredData = Iterable<List<double>>;
+typedef LandmarkVariations = List<List<double>>;
+
 class PoseDetector extends StatefulWidget {
   const PoseDetector({Key? key}) : super(key: key);
 
@@ -21,55 +24,46 @@ class PoseDetector extends StatefulWidget {
 }
 
 class _PoseDetectorState extends State<PoseDetector> {
+  static const buffer = 10;
+  static const _shouldWriteToFile = false;
+  final StreamController<Pose> _streamController = StreamController.broadcast();
+  final Directory appDir = Directory('/storage/emulated/0/Android/data/com.example.physigo/files');
   Image? _cameraImage;
   Pose? _detectedPose;
   Size _imageSize = Size.zero;
   late Future<void> _startCamera;
-  final StreamController<Pose> _streamController = StreamController.broadcast();
-  final Directory appDir = Directory('/storage/emulated/0/Android/data/com.example.physigo/files');
+  late Stream<LandmarkVariations> _variationsStream;
+  late Stream<MeanFilteredData> _meanFilterStream;
+  StreamController<Color> _stepExerciseStream = StreamController.broadcast();
 
   @override
   initState() {
     super.initState();
     _startCamera = _startCameraStream();
-    _writeDataToFile();
+    _meanFilterStream = _getMeanFilterStream(_streamController.stream);
+    if (_shouldWriteToFile) {
+      _writeDataToFile(_meanFilterStream);
+    }
+    _variationsStream = _meanFilterStream.pairwise().map(_calculateVariations);
+  }
+
+  LandmarkVariations _calculateVariations(Iterable<MeanFilteredData> pairPositions) {
+    final previous = pairPositions.first.toList();
+    final current = pairPositions.last.toList();
+    LandmarkVariations variations = [];
+    for (int landmark = 0; landmark < previous.length; landmark++) {
+      final dx = current[landmark][0] - previous[landmark][0];
+      final dy = current[landmark][1] - previous[landmark][1];
+      final dz = current[landmark][2] - previous[landmark][2];
+      variations.add([dx.roundToDouble(), dy.roundToDouble(), dz.roundToDouble()]);
+    }
+    return variations;
   }
 
-  void _writeDataToFile() {
-    const buffer = 10;
+  void _writeDataToFile(Stream<MeanFilteredData> stream) {
     File meanFilteredData = File("${appDir.path}/meanFilteredData.csv");
     if (meanFilteredData.existsSync()) meanFilteredData.deleteSync();
-    // TODO: get positions variation
-    // TODO: detect excentric/concentric part of the movement (derivative negative or positive)
-    _streamController.stream
-        .where((pose) => pose.landmarks.isNotEmpty)
-        .map((pose) => pose.landmarks.where((landmark) => authorizedType.contains(landmark.type)).toList())
-        .bufferCount(buffer, 1)
-        .listen((filteredLandmarks) {
-      // inverse height and width of the matrix
-      List<List<PoseLandmark>> bufferedPositions = [];
-      for (int j = 0; j < filteredLandmarks[0].length; j++) {
-        List<PoseLandmark> positions = [];
-        for (int i = 0; i < filteredLandmarks.length; i++) {
-          positions.add(filteredLandmarks[i][j]);
-        }
-        bufferedPositions.add(positions);
-      }
-      // mean filters of the buffer points
-      final meanPositions = bufferedPositions.map((landmarks) {
-        return landmarks
-            .map((landmark) => landmark.position)
-            .map((position) => [
-                  position.x / buffer,
-                  position.y / buffer,
-                  position.z / buffer,
-                ])
-            .reduce((value, element) => [
-                  value[0] + element[0],
-                  value[1] + element[1],
-                  value[2] + element[2],
-                ]);
-      });
+    stream.listen((meanPositions) {
       for (var position in meanPositions) {
         final str = "${position[0]}, ${position[1]}, ${position[2]};";
         meanFilteredData.writeAsStringSync(str, mode: FileMode.append);
@@ -78,6 +72,47 @@ class _PoseDetectorState extends State<PoseDetector> {
     });
   }
 
+  Stream<MeanFilteredData> _getMeanFilterStream(Stream<Pose> stream) {
+    return stream
+        .where((pose) => pose.landmarks.isNotEmpty)
+        .map((pose) => pose.landmarks.where((landmark) => authorizedType.contains(landmark.type)).toList())
+        // Get last [buffer] poses
+        .bufferCount(buffer, 1)
+        // Swap matrix [buffer] * [authorizedType.length]
+        .map(_swapMatrixDimensions)
+        // For every landmarks, get meanFilter of size [buffer]
+        .map((filteredLandmarks) => filteredLandmarks.map(_meanFilter));
+  }
+
+  List<double> _meanFilter(List<PoseLandmark> landmarks) {
+    return landmarks
+        .map((landmark) => landmark.position)
+        .map((position) => [
+              position.x / buffer,
+              position.y / buffer,
+              position.z / buffer,
+            ])
+        .reduce((value, element) => [
+              value[0] + element[0],
+              value[1] + element[1],
+              value[2] + element[2],
+            ]);
+  }
+
+  List<List<T>> _swapMatrixDimensions<T>(List<List<T>> matrix) {
+    final height = matrix.length;
+    final width = matrix[0].length;
+    List<List<T>> newMatrix = [];
+    for (int col = 0; col < width; col++) {
+      List<T> newRow = [];
+      for (int row = 0; row < height; row++) {
+        newRow.add(matrix[row][col]);
+      }
+      newMatrix.add(newRow);
+    }
+    return newMatrix;
+  }
+
   Future<void> _startCameraStream() async {
     await BodyDetection.startCameraStream(onFrameAvailable: _handleCameraImage, onPoseAvailable: _handlePose);
     await BodyDetection.enablePoseDetection();
@@ -131,15 +166,64 @@ class _PoseDetectorState extends State<PoseDetector> {
         if (snapshot.connectionState == ConnectionState.waiting) {
           return const Center(child: CircularProgressIndicator());
         }
-        return Center(
-          child: CustomPaint(
-            size: _imageSize,
-            child: _cameraImage,
-            foregroundPainter: PosePainter(
-              pose: _detectedPose,
-              imageSize: _imageSize,
+        return Column(
+          children: [
+            Center(
+              child: CustomPaint(
+                // size: _imageSize,
+                child: _cameraImage,
+                foregroundPainter: PosePainter(
+                  pose: _detectedPose,
+                  imageSize: _imageSize,
+                ),
+              ),
             ),
-          ),
+            StreamBuilder<MeanFilteredData>(
+              stream: _meanFilterStream,
+              builder: (context, snapshot) {
+                if (!snapshot.hasData) {
+                  return CircularProgressIndicator();
+                }
+                final landmarks = snapshot.data!.toList();
+                final xRightHip = landmarks[8][0];
+                final xRightKnee = landmarks[10][0];
+                final xDistanceHipKnee = (xRightHip - xRightKnee).abs();
+
+                final yRightHip = landmarks[8][1];
+                final yRightKnee = landmarks[10][1];
+                final yDistanceHipKnee = (yRightHip - yRightKnee).abs();
+                var message = "IN BETWEEN";
+                if (xDistanceHipKnee < 30) {
+                  message = "START";
+                  _stepExerciseStream.add(Colors.green);
+                } else if (yDistanceHipKnee < 40) {
+                  message = "END";
+                  _stepExerciseStream.add(Colors.red);
+                } else {
+                  _stepExerciseStream.add(Colors.yellow);
+                }
+
+                final zRightHip = landmarks[8][2];
+                final zRightKnee = landmarks[10][2];
+                final zDistanceHipKnee = (zRightHip - zRightKnee).abs();
+
+                return Text("$zDistanceHipKnee", style: TextStyle(fontSize: 40));
+              },
+            ),
+            StreamBuilder<Color>(
+              stream: _stepExerciseStream.stream,
+              builder: (context, snapshot) {
+                if (!snapshot.hasData) {
+                  return CircularProgressIndicator();
+                }
+                return Container(
+                  height: 100,
+                  width: 100,
+                  color: snapshot.data!,
+                );
+              },
+            ),
+          ],
         );
       },
     );
@@ -161,3 +245,17 @@ class _PoseDetectorState extends State<PoseDetector> {
     PoseLandmarkType.rightAnkle,
   ];
 }
+
+
+/*
+
+GETTING IN POSITION:
+  CHECK IF EVERY NECESSARY JOINT ARE ON SCREEN (reliability > 0.8)
+  CHECK IF START POSITION IS OKAY (for squat, if knee and hip on same x coordinate)
+
+COUTING REPETITION:
+  FROM BEGINNING TO END:
+    - BEGINNING: defined by start position, get position of interesting joint
+    - END: defined by positions/distance interesting joints (knee and hip same level for squat,
+          elbow and should same level for push up)
+ */

app/lib/exercises/exercises_validation/widgets/pose_painter.dart

@@ -27,13 +27,15 @@ class PosePainter extends CustomPainter {
     // Landmark connections
     final landmarksByType = {for (final it in pose!.landmarks) it.type: it};
     for (final connection in connections) {
-      if (landmarksByType[connection[0]] != null && landmarksByType[connection[1]] != null) {
+      if (landmarksByType[connection[0]] != null &&
+          landmarksByType[connection[1]] != null &&
+          landmarksByType[connection[0]]!.inFrameLikelihood > 0.5 &&
+          landmarksByType[connection[1]]!.inFrameLikelihood > 0.5) {
         final point1 = offsetForPart(landmarksByType[connection[0]]!);
         final point2 = offsetForPart(landmarksByType[connection[1]]!);
         canvas.drawLine(point1, point2, linePaint);
       }
     }
-
     // for (final part in pose!.landmarks) {
     //   // Landmark points
     //   canvas.drawCircle(offsetForPart(part), 5, pointPaint);